Prototype dc_blob_t API

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "daily"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 50
+    open-pull-requests-limit: 10

.github/mergeable.yml

@@ -1,26 +0,0 @@
-version: 2
-mergeable:
-  - when: pull_request.*
-    name: "Changelog check"
-    validate:
-      - do: or
-        validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/**'
-                required: ['CHANGELOG.md']
-    fail:
-      - do: checks
-        status: 'action_required' 
-        payload:
-          title: Changlog might need an update
-          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches
+          args: --workspace --tests --examples
 
   docs:
     name: Rust doc comments
@@ -77,19 +77,19 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.60.0
+            rust: 1.54.0
             python: 3.9
           - os: windows-latest
-            rust: 1.60.0
+            rust: 1.54.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.56.0
+          # Minimum Supported Rust Version = 1.51.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.56.0
+            rust: 1.51.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/dependabot.yml

@@ -1,21 +0,0 @@
-name: Dependabot auto-approve
-on: pull_request
-
-permissions:
-  pull-requests: write
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Approve a PR
-        run: gh pr review --approve "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/upload-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on rs.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat --no-deps
-    - name: Upload to rs.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/rs/"
-

.github/workflows/upload-ffi-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on cffi.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat_ffi --no-deps
-    - name: Upload to cffi.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/cffi/"
-

.gitignore

@@ -29,7 +29,3 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
-.vscode/launch.json
-python/accounts.txt
-python/all-testaccounts.txt
-tmp/

CHANGELOG.md

@@ -1,214 +1,11 @@
 # Changelog
 
-## 1.80.0
-
-### Changes
-- update provider database #3284
-- improve python bindings, tests and ci #3287 #3286 #3287 #3289 #3290 #3292
-
-### Fixes
-- fix escaping in generated QR-code-SVG #3295
-
-
-## 1.79.0
-
-### Changes
-- Send locations in the background regardless of SMTP loop activity #3247
-- refactorings #3268
-- improve tests and ci #3266 #3271
-
-### Fixes
-- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
-- fix correct message escaping consisting of a dot in SMTP protocol #3265
-
-
-## 1.78.0
-
-### API-Changes
-- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
-- add `dc_resend_msgs()` #3238
-- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
-  this is done by `dc_provider_new_from_email_with_dns()` now #3256
-
-### Changes
-- introduce multiple self addresses with the "configured" address always being the primary one #2896
-- Further improve finding the correct server after logging in #3208
-- `get_connectivity_html()` returns HTML as non-scalable #3213
-- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
-- Speed up message receiving via IMAP a bit #3225
-- mark messages as seen on IMAP in batches #3223
-- remove Received: based draft detection heuristic #3230
-- Use pkgconfig for building Python package #2590
-- don't start io on unconfigured context #2664
-- do not assign group IDs to ad-hoc groups #2798
-- dynamic libraries use dylib extension on Darwin #3226
-- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
-- improve documentation #3214 #3220 #3237
-- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
-
-### Fixes
-- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
-- Fix a bug where a blocked contact could send a contact request #3218
-- Make sure, videochat-room-names are always URL-safe #3231
-- Try removing account folder multiple times in case of failure #3229
-- Ignore messages from all spam folders if there are many #3246
-- Hide location-only messages instead of displaying empty bubbles #3248
-
-
-## 1.77.0
+## Unreleased
 
 ### API changes
-- change semantics of `dc_get_webxdc_status_updates()` second parameter
-  and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
-
-### Changes
-- add more SMTP logging #3093
-- place common headers like `From:` before the large `Autocrypt:` header #3079
-- keep track of securejoin joiner status in database to survive restarts #2920
-- remove never used `SentboxMove` option #3111
-- improve speed by caching config values #3131 #3145
-- optimize `markseen_msgs` #3141
-- automatically accept chats with outgoing messages #3143
-- `dc_receive_imf` refactorings #3154 #3156 #3159
-- add index to speedup deletion of expired ephemeral messages #3155
-- muted chats stay archived on new messages #3184
-- support `min_api` from Webxdc manifests #3206
-- do not read whole webxdc file into memory #3109
-- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
-- improve documentation #2983 #3112 #3103 #3118 #3120
-
-### Fixes
-- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
-- fix an issue where the app crashes when trying to export a backup #3195
-- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
-- do not delete messages without Message-IDs as duplicates #3095
-- assign replies from a different email address to the correct chat #3119
-- assing outgoing private replies to the correct chat #3177
-- start ephemeral timer when seen status is synchronized via IMAP #3122
-- do not create empty contact requests with "setup changed" messages;
-  instead, send a "setup changed" message into all chats we share with the peer #3187
-- do not delete duplicate messages on IMAP immediately to accidentally deleting
-  the last copy #3138
-- clear more columns when message expires due to `delete_device_after` setting #3181
-- do not try to use stale SMTP connections #3180
-- slightly improve finding the correct server after logging in #3207
-- retry message sending automatically if loop is not interrupted #3183
-- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
-
-
-## 1.76.0
-
-### Changes
-- move messages in batches #3058
-- delete messages in batches #3060
-- python: remove arbitrary timeouts from tests #3059
-- refactorings #3026
-
-### Fixes
-- avoid archived, fresh chats #3053
-- Also resync UIDs in folders that are not configured #2289
-- treat "NO" IMAP response to MOVE and COPY commands as an error #3058
-- Fix a bug where messages in the Spam folder created contact requests #3015
-- Fix a bug where drafts disappeared after some days #3067
-- Parse MS Exchange read receipts and mark the original message as read #3075
-- do not retry message sending infinitely in case of permanent SMTP failure #3070
-- set message state to failed when retry limit is exceeded #3072
-
-
-## 1.75.0
-
-### Changes
-- optimize `delete_expired_imap_messages()` #3047
-
-
-## 1.74.0
-
-### Fixes
-- avoid reconnection loop when message without Message-ID is marked as seen #3044
-
-
-## 1.73.0
-
-### API changes
-- added `only_fetch_mvbox` config #3028
-
-### Changes
-- don't watch Sent folder by default #3025
-- use webxdc app name in chatlist/quotes/replies etc. #3027
-- make it possible to cancel message sending by removing the message #3034,
-  this was previosuly removed in 1.71.0 #2939
-- synchronize Seen flags only on watched folders to speed up
-  folder scanning #3041
-- remove direct dependency on `byteorder` crate #3031
-- refactorings #3023 #3013
-- update provider database #3043
-- improve documentation #3017 #3018 #3021
-
-### Fixes
-- fix splitting off text from webxdc messages #3032
-- call slow `delete_expired_imap_messages()` less often #3037
-- make synchronization of Seen status more robust in case unsolicited FETCH
-  result without UID is returned #3022
-- fetch Inbox before scanning folders to ensure iOS does
-  not kill the app before it gets to fetch the Inbox in background #3040
-
-
-## 1.72.0
-
-### Fixes
-- run migrations on backup import #3006
-
-
-## 1.71.0
-
-### API Changes
-- added APIs to handle database passwords: `dc_context_new_closed()`, `dc_context_open()`,
-  `dc_context_is_open()` and `dc_accounts_add_closed_account()` #2956 #2972
-- use second parameter of `dc_imex` to provide backup passphrase #2980
-- added `DC_MSG_WEBXDC`, `dc_send_webxdc_status_update()`,
-  `dc_get_webxdc_status_updates()`, `dc_msg_get_webxdc_blob()`, `dc_msg_get_webxdc_info()`
-  and `DC_EVENT_WEBXDC_STATUS_UPDATE` #2826 #2971 #2975 #2977 #2979 #2993 #2994 #2998 #3001 #3003
-- added `dc_msg_get_parent()` #2984
-- added `dc_msg_force_plaintext()` API for bots #2847
-- allow removing quotes on drafts `dc_msg_set_quote(msg, NULL)` #2950
-- removed `mvbox_watch` option; watching is enabled when `mvbox_move` is enabled #2906
-- removed `inbox_watch` option #2922
-- deprecated `os_name` in `dc_context_new()`, pass `NULL` or an empty string #2956
-
-### Changes
-- start making it possible to write to mailing lists #2736
-- add `hop_info` to `dc_get_info()` #2751 #2914 #2923
-- add information about whether the database is encrypted or not to `dc_get_info()` #3000
-- selfstatus now defaults to empty #2951 #2960
-- validate detached cryptographic signatures as used eg. by Thunderbird #2865
-- do not change the draft's `msg_id` on updates and sending #2887
-- add `imap` table to keep track of message UIDs #2909 #2938
-- replace `SendMsgToSmtp` jobs which stored outgoing messages in blobdir with `smtp` SQL table #2939 #2996
-- sql: enable `auto_vacuum=INCREMENTAL` #2931
-- sql: build rusqlite with sqlcipher #2934
-- synchronize Seen status across devices #2942
-- `dc_preconfigure_keypair` now takes ascii armored keys instead of base64 #2862
-- put removed member in Bcc instead of To in the message about removal #2864
-- improve group updates #2889
-- re-write the blob filename creation loop #2981
-- update provider database (11 Jan 2022) #2959
-- python: allow timeout for internal configure tracker API #2967
-- python: remove API deprecated in Python 3.10 #2907
-- refactorings #2932 #2957 #2947
-- improve tests #2863 #2866 #2881 #2908 #2918 #2901 #2973
-- improve documentation #2880 #2886 #2895
-- improve ci #2919 #2926 #2969 #2999
-
-### Fixes
-- fix leaving groups #2929
-- fix unread count #2861
-- make `add_parts()` not early-exit #2879
-- recognize MS Exchange read receipts as read receipts #2890
-- create parent directory if creating a new file fails #2978
-- save "configured" flag later #2974
-- improve log #2928
-- `dc_receive_imf`: don't fail on invalid address in the To field #2940
 
+- Removed `mvbox_watch` option. #2906
+  It is automatically enabled whenever `mvbox_move` is enabled.
 
 ## 1.70.0
 

CMakeLists.txt

@@ -4,18 +4,10 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
-if(APPLE)
-       set(DYNAMIC_EXT "dylib")
-elseif(UNIX)
-       set(DYNAMIC_EXT "so")
-else()
-       set(DYNAMIC_EXT "dll")
-endif()
-
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
@@ -40,11 +32,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.80.0"
+version = "1.70.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2021"
+edition = "2018"
 license = "MPL-2.0"
-rust-version = "1.56"
+resolver = "2"
 
 [profile.dev]
 debug = 0
@@ -19,14 +19,15 @@ ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
 async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
-async-std-resolver = "0.21"
-async-std = { version = "1" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
+async-std-resolver = "0.20"
+async-std = { version = "1", features = ["unstable"] }
 async-tar = { version = "0.4", default-features=false }
 async-trait = "0.1"
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
+byteorder = "1.3"
 chrono = "0.4"
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
@@ -34,26 +35,26 @@ encoded-words = { git = "https://github.com/async-email/encoded-words", branch="
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
-image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-log = {version = "0.4.16", optional = true }
+log = {version = "0.4.8", optional = true }
 mailparse = "0.13"
 native-tls = "0.2"
 num_cpus = "1.13"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.10.0"
+once_cell = "1.9.0"
 percent-encoding = "2.0"
 pgp = { version = "0.7", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
 quick-xml = "0.22"
 r2d2 = "0.8"
-r2d2_sqlite = "0.20"
+r2d2_sqlite = "0.19"
 rand = "0.7"
 regex = "1.5"
-rusqlite = { version = "0.27", features = ["sqlcipher"] }
+rusqlite = { version = "0.26", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 rustyline = { version = "9", optional = true }
 sanitize-filename = "0.3"
@@ -62,8 +63,9 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-strum = "0.24"
-strum_macros = "0.24"
+stop-token = "0.6"
+strum = "0.23"
+strum_macros = "0.23"
 surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
@@ -72,14 +74,13 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
 fast-socks5 = "0.4"
 humansize = "1"
 qrcodegen = "1.7.0"
-tagger = "4.3.3"
-textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
+tagger = "3.3.0"
+textwrap = "0.14.2"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
 async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = { version = "0.3.4", features = ["async_std"] }
+criterion = "0.3"
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
@@ -115,18 +116,6 @@ harness = false
 name = "search_msgs"
 harness = false
 
-[[bench]]
-name = "receive_emails"
-harness = false
-
-[[bench]]
-name = "get_chat_msgs"
-harness = false
-
-[[bench]]
-name = "get_chatlist"
-harness = false
-
 [features]
 default = ["vendored"]
 internals = []

README.md

@@ -12,8 +12,6 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
-> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
-
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:
@@ -127,11 +125,11 @@ $ cargo test -- --ignored
 
 Language bindings are available for:
 
-- **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
-- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
-- **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/deltachat/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

assets/icon-webxdc.svg

@@ -1,181 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="80mm"
-   height="297mm"
-   viewBox="0 0 80 297"
-   version="1.1"
-   id="svg71"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   sodipodi:docname="icon-webxdc.svg"
-   inkscape:export-filename="C:\Users\user\OneDrive - BFW-Leipzig\Documents\LogoDC\finalohnerand.png"
-   inkscape:export-xdpi="96"
-   inkscape:export-ydpi="96">
-  <metadata
-     id="metadata856">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <sodipodi:namedview
-     id="namedview73"
-     pagecolor="#767676"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="true"
-     inkscape:document-units="mm"
-     showgrid="false"
-     showborder="false"
-     inkscape:snap-bbox="true"
-     inkscape:bbox-paths="true"
-     inkscape:bbox-nodes="true"
-     inkscape:snap-bbox-edge-midpoints="true"
-     inkscape:snap-bbox-midpoints="true"
-     inkscape:object-paths="true"
-     inkscape:snap-intersection-paths="true"
-     inkscape:zoom="1.4142136"
-     inkscape:cx="-90.271136"
-     inkscape:cy="-1233.1209"
-     inkscape:window-width="1864"
-     inkscape:window-height="1027"
-     inkscape:window-x="56"
-     inkscape:window-y="25"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="layer1"
-     inkscape:snap-global="false"
-     showguides="false"
-     inkscape:guide-bbox="true"
-     inkscape:document-rotation="0"
-     units="px">
-    <sodipodi:guide
-       position="-154.76097,641.11689"
-       orientation="0,-1"
-       id="guide21118" />
-    <sodipodi:guide
-       position="-60.286487,633.36619"
-       orientation="0,-1"
-       id="guide21120" />
-  </sodipodi:namedview>
-  <defs
-     id="defs68">
-    <linearGradient
-       id="linearGradient4375">
-      <stop
-         style="stop-color:#364e59;stop-opacity:1;"
-         offset="0"
-         id="stop4377" />
-      <stop
-         style="stop-color:#364e59;stop-opacity:0;"
-         offset="1"
-         id="stop4379" />
-    </linearGradient>
-  </defs>
-  <g
-     inkscape:label="Ebene 1"
-     inkscape:groupmode="layer"
-     id="layer1">
-    <rect
-       style="fill:#1a1a1a;stroke:#000000;stroke-width:0.167903"
-       id="rect880"
-       width="79.8321"
-       height="79.8321"
-       x="-64.03286"
-       y="-375.9097"
-       ry="0" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path3799-2"
-       d="m -24.089585,-372.59579 c -19.986026,0.24336 -36.196903,16.666 -36.196903,36.67011 0,20.00409 16.210877,36.03233 36.196903,35.78912 19.0024236,-0.076 14.5340713,-10.6146 35.538854,-0.85693 -11.50627538,-17.97454 0.390097,-20.36737 0.658079,-35.81316 0,-20.00411 -16.2108788,-36.03235 -36.196911,-35.78914 z"
-       style="fill:#364e59;fill-opacity:1;stroke:none;stroke-width:1.93355"
-       sodipodi:nodetypes="sscccs" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -54.193871,-325.26419 Z"
-       id="path3846" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.397951,-326.67773 Z"
-       id="path3848" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="m -49.397951,-326.67773 v 0 0"
-       id="path3850" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.01;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
-       d="m -51.35133,-325.0334 -7.964067,5.98895 z"
-       id="path3965" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path11037"
-       d="m -24.089585,-372.19891 c -19.986026,0.24156 -36.196903,16.54352 -36.196903,36.40062 0,7.86524 2.543315,15.1113 6.857155,20.97971 6.577146,8.94734 11.123515,9.77363 11.123515,9.77363 1.343237,1.78324 10.270932,4.3223 10.270932,4.3223 l 16.791727,-70.86654 -0.468369,-0.33457 c 0.458597,0.26445 0.428277,-0.27515 -8.378035,-0.27515 z"
-       style="fill:#7cc5cc;fill-opacity:1;stroke:none;stroke-width:1.92643"
-       sodipodi:nodetypes="sssccccss" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.944239,-310.69957 Z"
-       id="path13674" />
-    <g
-       id="g15178"
-       transform="matrix(0.79975737,0,0,0.79975737,53.088959,-63.716396)">
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072"
-         width="29.897917"
-         height="6.8791666"
-         x="-334.4964"
-         y="-154.51025"
-         transform="rotate(45)" />
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072-5"
-         width="29.897917"
-         height="6.8791666"
-         x="147.63107"
-         y="-334.4964"
-         transform="rotate(-45)"
-         inkscape:transform-center-x="-0.74835017"
-         inkscape:transform-center-y="0.37417525" />
-    </g>
-    <g
-       id="g22468"
-       transform="translate(3.3033974)">
-      <g
-         id="g15178-0"
-         transform="matrix(-0.79975737,0,0,0.79975737,-103.11028,-63.716404)"
-         style="fill:#7cc5cc;fill-opacity:1">
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-2"
-           width="29.897917"
-           height="6.8791666"
-           x="-334.4964"
-           y="-154.51025"
-           transform="rotate(45)" />
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-5-5"
-           width="29.897917"
-           height="6.8791666"
-           x="147.63107"
-           y="-334.4964"
-           transform="rotate(-45)"
-           inkscape:transform-center-x="-0.74835017"
-           inkscape:transform-center-y="0.37417525" />
-      </g>
-    </g>
-  </g>
-</svg>

benches/get_chat_msgs.rs

@@ -1,40 +0,0 @@
-use async_std::path::Path;
-
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chat::{self, ChatId};
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    for c in chats.iter().take(10) {
-        black_box(chat::get_chat_msgs(&context, *c, 0, None).await.ok());
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
-            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
-            let len = chatlist.len();
-            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
-        });
-
-        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/get_chatlist.rs

@@ -1,27 +0,0 @@
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_list_benchmark(context: &Context) {
-    Chatlist::try_load(context, 0, None, None).await.unwrap();
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
-        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_list_benchmark(black_box(&context)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/receive_emails.rs

@@ -1,84 +0,0 @@
-use async_std::{path::PathBuf, task::block_on};
-use criterion::{
-    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
-    Criterion,
-};
-use deltachat::{
-    config::Config,
-    context::Context,
-    dc_receive_imf::dc_receive_imf,
-    imex::{imex, ImexMode},
-};
-use tempfile::tempdir;
-
-async fn recv_all_emails(context: Context) -> Context {
-    for i in 0..100 {
-        let imf_raw = format!(
-            "Subject: Benchmark
-Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
-Date: Sat, 07 Dec 2019 19:00:27 +0000
-To: alice@example.com
-From: sender@testrun.org
-Chat-Version: 1.0
-Chat-Disposition-Notification-To: sender@testrun.org
-Chat-User-Avatar: 0
-In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
-MIME-Version: 1.0
-
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Hello {i}",
-            i = i,
-            i_dec = i - 1,
-        );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
-            .await
-            .unwrap();
-    }
-    context
-}
-
-async fn create_context() -> Context {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    let backup: PathBuf = std::env::current_dir()
-        .unwrap()
-        .join("delta-chat-backup.tar")
-        .into();
-    if backup.exists().await {
-        println!("Importing backup");
-        imex(&context, ImexMode::ImportBackup, &backup, None)
-            .await
-            .unwrap();
-    }
-
-    let addr = "alice@example.com";
-    context.set_config(Config::Addr, Some(addr)).await.unwrap();
-    context
-        .set_config(Config::ConfiguredAddr, Some(addr))
-        .await
-        .unwrap();
-    context
-        .set_config(Config::Configured, Some("1"))
-        .await
-        .unwrap();
-    context
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    let mut group = c.benchmark_group("Receive messages");
-    group.bench_function("Receive 100 simple text msgs", |b| {
-        b.to_async(AsyncStdExecutor).iter_batched(
-            || block_on(create_context()),
-            |context| recv_all_emails(black_box(context)),
-            BatchSize::LargeInput,
-        );
-    });
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -20,8 +20,6 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.80.0"
+version = "1.70.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/README.md

@@ -1,9 +1,5 @@
 # Delta Chat C Interface
 
-## Installation
-
-see `Installing libdeltachat system wide` in [../README.md](../README.md)
-
 ## Documentation
 
 To generate the C Interface documentation,

deltachat-ffi/src/lib.rs

@@ -27,17 +27,15 @@ use async_std::sync::RwLock;
 use async_std::task::{block_on, spawn};
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use num_traits::{FromPrimitive, ToPrimitive};
-use rand::Rng;
 
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::{Contact, ContactId, Origin};
+use deltachat::contact::{Contact, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
 use deltachat::stock_str::StockMessage;
-use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 
@@ -77,6 +75,7 @@ pub unsafe extern "C" fn dc_context_new(
     }
 
     let ctx = if blobdir.is_null() || *blobdir == 0 {
+        use rand::Rng;
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
         block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
@@ -87,63 +86,12 @@ pub unsafe extern "C" fn dc_context_new(
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
         Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
+            eprintln!("failed to create context: {}", err);
             ptr::null_mut()
         }
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *mut dc_context_t {
-    setup_panic!();
-
-    if dbfile.is_null() {
-        eprintln!("ignoring careless call to dc_context_new_closed()");
-        return ptr::null_mut();
-    }
-
-    let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile).to_path_buf().into(),
-        id,
-    )) {
-        Ok(context) => Box::into_raw(Box::new(context)),
-        Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
-            ptr::null_mut()
-        }
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_open(
-    context: *mut dc_context_t,
-    passphrase: *const libc::c_char,
-) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_context_open()");
-        return 0;
-    }
-
-    let ctx = &*context;
-    let passphrase = to_string_lossy(passphrase);
-    block_on(ctx.open(passphrase))
-        .log_err(ctx, "dc_context_open() failed")
-        .map(|b| b as libc::c_int)
-        .unwrap_or(0)
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_context_is_open(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_context_is_open()");
-        return 0;
-    }
-
-    let ctx = &*context;
-    block_on(ctx.is_open()) as libc::c_int
-}
-
 /// Release the context structure.
 ///
 /// This function releases the memory of the `dc_context_t` structure.
@@ -493,17 +441,14 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ChatEphemeralTimerModified { chat_id, .. } => chat_id.to_u32() as libc::c_int,
         EventType::ContactsChanged(id) | EventType::LocationChanged(id) => {
             let id = id.unwrap_or_default();
-            id.to_u32() as libc::c_int
+            id as libc::c_int
         }
         EventType::ConfigureProgress { progress, .. } | EventType::ImexProgress(progress) => {
             *progress as libc::c_int
         }
         EventType::ImexFileWritten(_) => 0,
         EventType::SecurejoinInviterProgress { contact_id, .. }
-        | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
-            contact_id.to_u32() as libc::c_int
-        }
-        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        | EventType::SecurejoinJoinerProgress { contact_id, .. } => *contact_id as libc::c_int,
     }
 }
 
@@ -535,8 +480,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
-        | EventType::SelfavatarChanged => 0,
-        EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged
+        | EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
@@ -545,10 +490,6 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::SecurejoinInviterProgress { progress, .. }
         | EventType::SecurejoinJoinerProgress { progress, .. } => *progress as libc::c_int,
         EventType::ChatEphemeralTimerModified { timer, .. } => timer.to_u32() as libc::c_int,
-        EventType::WebxdcStatusUpdate {
-            status_update_serial,
-            ..
-        } => status_update_serial.to_u32() as libc::c_int,
     }
 }
 
@@ -590,7 +531,6 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SecurejoinJoinerProgress { .. }
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
-        | EventType::WebxdcStatusUpdate { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -657,7 +597,7 @@ pub unsafe extern "C" fn dc_get_next_event(events: *mut dc_event_emitter_t) -> *
 #[no_mangle]
 pub unsafe extern "C" fn dc_stop_io(context: *mut dc_context_t) {
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_stop_io()");
+        eprintln!("ignoring careless call to dc_shutdown()");
         return;
     }
     let ctx = &*context;
@@ -720,11 +660,7 @@ pub unsafe extern "C" fn dc_get_chatlist(
     let ctx = &*context;
     let qs = to_opt_string_lossy(query_str);
 
-    let qi = if query_id == 0 {
-        None
-    } else {
-        Some(ContactId::new(query_id))
-    };
+    let qi = if query_id == 0 { None } else { Some(query_id) };
 
     block_on(async move {
         match chatlist::Chatlist::try_load(ctx, flags as usize, qs.as_deref(), qi)
@@ -752,7 +688,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::create_for_contact(ctx, ContactId::new(contact_id))
+        ChatId::create_for_contact(ctx, contact_id)
             .await
             .log_err(ctx, "Failed to create chat from contact_id")
             .map(|id| id.to_u32())
@@ -772,7 +708,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        ChatId::lookup_by_contact(ctx, ContactId::new(contact_id))
+        ChatId::lookup_by_contact(ctx, contact_id)
             .await
             .log_err(ctx, "Failed to get chat for contact_id")
             .unwrap_or_default() // unwraps the Result
@@ -884,48 +820,6 @@ pub unsafe extern "C" fn dc_send_videochat_invitation(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_send_webxdc_status_update(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    json: *const libc::c_char,
-    descr: *const libc::c_char,
-) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_send_webxdc_status_update()");
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.send_webxdc_status_update(
-        MsgId::new(msg_id),
-        &to_string_lossy(json),
-        &to_string_lossy(descr),
-    ))
-    .log_err(ctx, "Failed to send webxdc update")
-    .is_ok() as libc::c_int
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_webxdc_status_updates(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    last_known_serial: u32,
-) -> *mut libc::c_char {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_get_webxdc_status_updates()");
-        return "".strdup();
-    }
-    let ctx = &*context;
-
-    block_on(ctx.get_webxdc_status_updates(
-        MsgId::new(msg_id),
-        StatusUpdateSerial::new(last_known_serial),
-    ))
-    .unwrap_or_else(|_| "".to_string())
-    .strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -1349,10 +1243,7 @@ pub unsafe extern "C" fn dc_get_chat_contacts(
         let arr = dc_array_t::from(
             chat::get_chat_contacts(ctx, ChatId::new(chat_id))
                 .await
-                .unwrap_or_log_default(ctx, "Failed get_chat_contacts")
-                .iter()
-                .map(|id| id.to_u32())
-                .collect::<Vec<u32>>(),
+                .unwrap_or_log_default(ctx, "Failed get_chat_contacts"),
         );
         Box::into_raw(Box::new(arr))
     })
@@ -1462,7 +1353,7 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     block_on(chat::is_contact_in_chat(
         ctx,
         ChatId::new(chat_id),
-        ContactId::new(contact_id),
+        contact_id,
     ))
     .log_err(ctx, "is_contact_in_chat failed")
     .unwrap_or_default() as libc::c_int
@@ -1483,7 +1374,7 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     block_on(chat::add_contact_to_chat(
         ctx,
         ChatId::new(chat_id),
-        ContactId::new(contact_id),
+        contact_id,
     ))
     .log_err(ctx, "Failed to add contact")
     .is_ok() as libc::c_int
@@ -1504,7 +1395,7 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     block_on(chat::remove_contact_from_chat(
         ctx,
         ChatId::new(chat_id),
-        ContactId::new(contact_id),
+        contact_id,
     ))
     .log_err(ctx, "Failed to remove contact")
     .is_ok() as libc::c_int
@@ -1751,27 +1642,6 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_resend_msgs(
-    context: *mut dc_context_t,
-    msg_ids: *const u32,
-    msg_cnt: libc::c_int,
-) -> libc::c_int {
-    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
-        eprintln!("ignoring careless call to dc_resend_msgs()");
-        return 0;
-    }
-    let ctx = &*context;
-    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
-
-    if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
-        error!(ctx, "Resending failed: {}", err);
-        0
-    } else {
-        1
-    }
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_markseen_msgs(
     context: *mut dc_context_t,
@@ -1860,8 +1730,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
         Contact::lookup_id_by_addr(ctx, &to_string_lossy(addr), Origin::IncomingReplyTo)
             .await
             .unwrap_or_log_default(ctx, "failed to lookup id")
-            .map(|id| id.to_u32())
-            .unwrap_or_default()
+            .unwrap_or(0)
     })
 }
 
@@ -1881,7 +1750,6 @@ pub unsafe extern "C" fn dc_create_contact(
     block_on(async move {
         Contact::create(ctx, &name, &to_string_lossy(addr))
             .await
-            .map(|id| id.to_u32())
             .unwrap_or(0)
     })
 }
@@ -1920,9 +1788,7 @@ pub unsafe extern "C" fn dc_get_contacts(
 
     block_on(async move {
         match Contact::get_all(ctx, flags, query).await {
-            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(
-                contacts.iter().map(|id| id.to_u32()).collect::<Vec<u32>>(),
-            ))),
+            Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
             Err(_) => ptr::null_mut(),
         }
     })
@@ -1959,10 +1825,7 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
             Contact::get_all_blocked(ctx)
                 .await
                 .log_err(ctx, "Can't get blocked contacts")
-                .unwrap_or_default()
-                .iter()
-                .map(|id| id.to_u32())
-                .collect::<Vec<u32>>(),
+                .unwrap_or_default(),
         )))
     })
 }
@@ -1973,8 +1836,7 @@ pub unsafe extern "C" fn dc_block_contact(
     contact_id: u32,
     block: libc::c_int,
 ) {
-    let contact_id = ContactId::new(contact_id);
-    if context.is_null() || contact_id.is_special() {
+    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
         eprintln!("ignoring careless call to dc_block_contact()");
         return;
     }
@@ -2004,7 +1866,7 @@ pub unsafe extern "C" fn dc_get_contact_encrinfo(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_encrinfo(ctx, ContactId::new(contact_id))
+        Contact::get_encrinfo(ctx, contact_id)
             .await
             .map(|s| s.strdup())
             .unwrap_or_else(|e| {
@@ -2019,8 +1881,7 @@ pub unsafe extern "C" fn dc_delete_contact(
     context: *mut dc_context_t,
     contact_id: u32,
 ) -> libc::c_int {
-    let contact_id = ContactId::new(contact_id);
-    if context.is_null() || contact_id.is_special() {
+    if context.is_null() || contact_id <= constants::DC_CONTACT_ID_LAST_SPECIAL as u32 {
         eprintln!("ignoring careless call to dc_delete_contact()");
         return 0;
     }
@@ -2046,7 +1907,7 @@ pub unsafe extern "C" fn dc_get_contact(
     let ctx = &*context;
 
     block_on(async move {
-        Contact::get_by_id(ctx, ContactId::new(contact_id))
+        Contact::get_by_id(ctx, contact_id)
             .await
             .map(|contact| Box::into_raw(Box::new(ContactWrapper { context, contact })))
             .unwrap_or_else(|_| ptr::null_mut())
@@ -2058,7 +1919,7 @@ pub unsafe extern "C" fn dc_imex(
     context: *mut dc_context_t,
     what_raw: libc::c_int,
     param1: *const libc::c_char,
-    param2: *const libc::c_char,
+    _param2: *const libc::c_char,
 ) {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_imex()");
@@ -2071,13 +1932,12 @@ pub unsafe extern "C" fn dc_imex(
             return;
         }
     };
-    let passphrase = to_opt_string_lossy(param2);
 
     let ctx = &*context;
 
     if let Some(param1) = to_opt_string_lossy(param1) {
         spawn(async move {
-            imex::imex(ctx, what, param1.as_ref(), passphrase)
+            imex::imex(ctx, what, param1.as_ref())
                 .await
                 .log_err(ctx, "IMEX failed")
         });
@@ -2469,7 +2329,7 @@ pub unsafe extern "C" fn dc_array_get_contact_id(
         return 0;
     }
 
-    (*array).get_location(index).contact_id.to_u32()
+    (*array).get_location(index).contact_id
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_msg_id(
@@ -2583,14 +2443,7 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    let ctx = &*ffi_list.context;
-    match ffi_list.list.get_chat_id(index as usize) {
-        Ok(chat_id) => chat_id.to_u32(),
-        Err(err) => {
-            warn!(ctx, "get_chat_id failed: {}", err);
-            0
-        }
-    }
+    ffi_list.list.get_chat_id(index as usize).to_u32()
 }
 
 #[no_mangle]
@@ -2985,7 +2838,7 @@ pub unsafe extern "C" fn dc_msg_get_from_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_from_id().to_u32()
+    ffi_msg.message.get_from_id()
 }
 
 #[no_mangle]
@@ -3107,61 +2960,6 @@ pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c
     ffi_msg.message.get_filename().unwrap_or_default().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_webxdc_blob(
-    msg: *mut dc_msg_t,
-    filename: *const libc::c_char,
-    ret_bytes: *mut libc::size_t,
-) -> *mut libc::c_char {
-    if msg.is_null() || filename.is_null() || ret_bytes.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_webxdc_blob()");
-        return ptr::null_mut();
-    }
-    let ffi_msg = &*msg;
-    let ctx = &*ffi_msg.context;
-    let blob = block_on(async move {
-        ffi_msg
-            .message
-            .get_webxdc_blob(ctx, &to_string_lossy(filename))
-            .await
-    });
-    match blob {
-        Ok(blob) => {
-            *ret_bytes = blob.len();
-            let ptr = libc::malloc(*ret_bytes);
-            libc::memcpy(ptr, blob.as_ptr() as *mut libc::c_void, *ret_bytes);
-            ptr as *mut libc::c_char
-        }
-        Err(err) => {
-            eprintln!("failed read blob from archive: {}", err);
-            ptr::null_mut()
-        }
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_webxdc_info(msg: *mut dc_msg_t) -> *mut libc::c_char {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_webxdc_info()");
-        return "".strdup();
-    }
-    let ffi_msg = &*msg;
-    let ctx = &*ffi_msg.context;
-
-    block_on(async move {
-        let info = match ffi_msg.message.get_webxdc_info(ctx).await {
-            Ok(info) => info,
-            Err(err) => {
-                error!(ctx, "dc_msg_get_webxdc_info() failed to get info: {}", err);
-                return "".strdup();
-            }
-        };
-        serde_json::to_string(&info)
-            .unwrap_or_log_default(ctx, "dc_msg_get_webxdc_info() failed to serialise to json")
-            .strdup()
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -3574,21 +3372,17 @@ pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_m
         return;
     }
     let ffi_msg = &mut *msg;
-    let quote_msg = if quote.is_null() {
-        None
-    } else {
-        let ffi_quote = &*quote;
-        if ffi_msg.context != ffi_quote.context {
-            eprintln!("ignoring attempt to quote message from a different context");
-            return;
-        }
-        Some(&ffi_quote.message)
-    };
+    let ffi_quote = &*quote;
+
+    if ffi_msg.context != ffi_quote.context {
+        eprintln!("ignoring attempt to quote message from a different context");
+        return;
+    }
 
     block_on(async move {
         ffi_msg
             .message
-            .set_quote(&*ffi_msg.context, quote_msg)
+            .set_quote(&*ffi_msg.context, &ffi_quote.message)
             .await
             .log_err(&*ffi_msg.context, "failed to set quote")
             .ok();
@@ -3631,29 +3425,6 @@ pub unsafe extern "C" fn dc_msg_get_quoted_msg(msg: *const dc_msg_t) -> *mut dc_
     }
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_get_parent(msg: *const dc_msg_t) -> *mut dc_msg_t {
-    if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_get_parent()");
-        return ptr::null_mut();
-    }
-    let ffi_msg: &MessageWrapper = &*msg;
-    let context = &*ffi_msg.context;
-    let res = block_on(async move {
-        ffi_msg
-            .message
-            .parent(context)
-            .await
-            .log_err(context, "failed to get parent message")
-            .unwrap_or(None)
-    });
-
-    match res {
-        Some(message) => Box::into_raw(Box::new(MessageWrapper { context, message })),
-        None => ptr::null_mut(),
-    }
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
     if msg.is_null() {
@@ -3696,7 +3467,7 @@ pub unsafe extern "C" fn dc_contact_get_id(contact: *mut dc_contact_t) -> u32 {
         return 0;
     }
     let ffi_contact = &*contact;
-    ffi_contact.contact.get_id().to_u32()
+    ffi_contact.contact.get_id()
 }
 
 #[no_mangle]
@@ -3973,25 +3744,6 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
     }
     let addr = to_string_lossy(addr);
 
-    let ctx = &*context;
-
-    match block_on(provider::get_provider_info(ctx, addr.as_str(), true)) {
-        Some(provider) => provider,
-        None => ptr::null_mut(),
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_provider_new_from_email_with_dns(
-    context: *const dc_context_t,
-    addr: *const libc::c_char,
-) -> *const dc_provider_t {
-    if context.is_null() || addr.is_null() {
-        eprintln!("ignoring careless call to dc_provider_new_from_email_with_dns()");
-        return ptr::null();
-    }
-    let addr = to_string_lossy(addr);
-
     let ctx = &*context;
     let socks5_enabled = block_on(async move {
         ctx.get_config_bool(config::Config::Socks5Enabled)
@@ -4202,30 +3954,6 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_accounts_add_closed_account(accounts: *mut dc_accounts_t) -> u32 {
-    if accounts.is_null() {
-        eprintln!("ignoring careless call to dc_accounts_add_closed_account()");
-        return 0;
-    }
-
-    let accounts = &mut *accounts;
-
-    block_on(async move {
-        let mut accounts = accounts.write().await;
-        match accounts.add_closed_account().await {
-            Ok(id) => id,
-            Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to add account: {:#}",
-                    err
-                )));
-                0
-            }
-        }
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_remove_account(
     accounts: *mut dc_accounts_t,

deltachat-ffi/src/lot.rs

@@ -111,19 +111,19 @@ impl Lot {
         match self {
             Self::Summary(_) => Default::default(),
             Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
+                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
                 Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => contact_id.to_u32(),
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
+                Qr::FprOk { contact_id } => *contact_id,
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
+                Qr::Addr { contact_id } => *contact_id,
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
+                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
                 Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
+                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
                 Qr::ReviveVerifyGroup { .. } => Default::default(),
             },
             Self::Error(_) => Default::default(),

draft/webxdc-dev-reference.md

@@ -1,185 +0,0 @@
-# Webxdc Developer Reference
-
-## Webxdc File Format
-
-- a **Webxdc app** is a **ZIP-file** with the extension `.xdc`
-- the ZIP-file must use the default compression methods as of RFC 1950,
-  this is "Deflate" or "Store"
-- the ZIP-file must contain at least the file `index.html`
-- if the Webxdc app is started, `index.html` is opened in a restricted webview
-  that allow accessing resources only from the ZIP-file
-
-
-## Webxdc API
-
-There are some additional APIs available once `webxdc.js` is included
-(the file will be provided by the concrete implementations,
-no need to add `webxdc.js` to your ZIP-file):
-
-```html
-<script src="webxdc.js"></script>
-```
-
-### sendUpdate()
-
-```js
-window.webxdc.sendUpdate(update, descr);
-```
-
-Webxdc apps are usually shared in a chat and run independently on each peer.
-To get a shared state, the peers use `sendUpdate()` to send updates to each other.
-
-- `update`: an object with the following properties:  
-    - `update.payload`: any javascript primitive, array or object.
-    - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
-       use this option sparingly to not spam the chat.
-    - `update.summary`: optional, short text, shown beside app icon;
-       it is recommended to use some aggregated value,  eg. "8 votes", "Highscore: 123"
-
-- `descr`: short, human-readable description what this update is about.
-  this is shown eg. as a fallback text in an email program.
-
-All peers, including the sending one,
-will receive the update by the callback given to `setUpdateListener()`.
-
-There are situations where the user cannot send messages to a chat,
-eg. if the webxdc instance comes as a contact request or if the user has left a group.
-In these cases, you can still call `sendUpdate()`,
-however, the update won't be sent to other peers
-and you won't get the update by `setUpdateListener()`.
-
-
-### setUpdateListener()
-
-```js
-let promise = window.webxdc.setUpdateListener((update) => {}, serial);
-```
-
-With `setUpdateListener()` you define a callback that receives the updates
-sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
-The `serial` specifies the last serial that you know about (defaults to 0). 
-The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
-
-Each `update` which is passed to the callback comes with the following properties: 
-
-- `update.payload`: equals the payload given to `sendUpdate()`
-
-- `update.serial`: the serial number of this update.
-  Serials are larger `0` and newer serials have higher numbers.
-  There may be gaps in the serials
-  and it is not guaranteed that the next serial is exactly incremented by one.
-
-- `update.max_serial`: the maximum serial currently known.
-  If `max_serial` equals `serial` this update is the last update (until new network messages arrive).
-
-- `update.info`: optional, short, informational message (see `send_update`) 
-
-- `update.summary`: optional, short text, shown beside app icon (see `send_update`)
-
-
-### selfAddr
-
-```js
-window.webxdc.selfAddr
-```
-
-Property with the peer's own address.
-This is esp. useful if you want to differ between different peers -
-just send the address along with the payload,
-and, if needed, compare the payload addresses against selfAddr() later on.
-
-
-### selfName
-
-```js
-window.webxdc.selfName
-```
-
-Property with the peer's own name.
-This is name chosen by the user in their settings,
-if there is nothing set, that defaults to the peer's address.
-
-
-## manifest.toml
-
-If the ZIP-file contains a `manifest.toml` in its root directory,
-some basic information are read and used from there.
-
-the `manifest.toml` has the following format
-
-```toml
-name = "My App Name"
-```
-
-- **name** - The name of the app.
-  If no name is set or if there is no manifest, the filename is used as the app name.
-
-
-## App Icon
-
-If the ZIP-root contains an `icon.png` or `icon.jpg`,
-these files are used as the icon for the app.
-The icon should be a square at reasonable width/height;
-round corners etc. will be added by the implementations as needed.
-If no icon is set, a default icon will be used.
-
-
-## Webxdc Examples
-
-The following example shows an input field and  every input is show on all peers.
-
-```html
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8"/>
-    <script src="webxdc.js"></script>
-  </head>
-  <body>
-    <input id="input" type="text"/>
-    <a href="" onclick="sendMsg(); return false;">Send</a>
-    <p id="output"></p>
-    <script>
-    
-      function sendMsg() {
-        msg = document.getElementById("input").value;
-        window.webxdc.sendUpdate({payload: msg}, 'Someone typed "'+msg+'".');
-      }
-    
-      function receiveUpdate(update) {
-        document.getElementById('output').innerHTML += update.payload + "<br>";
-      }
-    
-      window.webxdc.setUpdateListener(receiveUpdate, 0);
-    </script>
-  </body>
-</html>
-```
-
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
-You can also use that repository as a template for your own app -
-just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -17,10 +17,11 @@ use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
+use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -83,7 +84,6 @@ async fn reset_tables(context: &Context, bits: i32) {
             )
             .await
             .unwrap();
-        context.sql().config_cache().write().await.clear();
         context
             .sql()
             .execute("DELETE FROM leftgrps;", paramsv![])
@@ -92,13 +92,16 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_msgs_changed_without_ids();
+    context.emit_event(EventType::MsgsChanged {
+        chat_id: ChatId::new(0),
+        msg_id: MsgId::new(0),
+    });
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, false).await {
+    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -160,7 +163,10 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_msgs_changed_without_ids();
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
     }
     true
 }
@@ -203,7 +209,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == ContactId::SELF {
+        if msg.get_from_id() == 1 {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -261,8 +267,9 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
     for contact_id in contacts {
+        let line;
         let mut line2 = "".to_string();
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
@@ -277,20 +284,24 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         } else {
             ""
         };
-        let line = format!(
+        line = format!(
             "{}{} <{}>",
             if !name.is_empty() {
-                name
+                &name
             } else {
                 "<name unset>"
             },
             verified_str,
-            if !addr.is_empty() { addr } else { "addr unset" }
+            if !addr.is_empty() {
+                &addr
+            } else {
+                "addr unset"
+            }
         );
-        let peerstate = Peerstate::from_addr(context, addr)
+        let peerstate = Peerstate::from_addr(context, &addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != ContactId::SELF {
+        if peerstate.is_some() && *contact_id != 1 {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -376,7 +387,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
                  sendsyncmsg\n\
-                 sendupdate <msg-id> <json status update>\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -399,7 +409,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
-                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -420,6 +429,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
+                 event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
                  clear -- clear screen\n\
@@ -461,32 +471,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(
-                &context,
-                ImexMode::ExportBackup,
-                dir.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(
-                &context,
-                ImexMode::ImportBackup,
-                arg1.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -565,7 +563,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
                         "{}#{}: {} [{} fresh] {}{}{}{}",
                         chat_prefix(&chat),
@@ -708,7 +706,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
+            let contact_id: u32 = arg1.parse()?;
             let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
@@ -736,7 +734,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0 = ContactId::new(arg1.parse()?);
+            let contact_id_0: u32 = arg1.parse()?;
             chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
                 .await?;
             println!("Contact added to chat.");
@@ -744,7 +742,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1 = ContactId::new(arg1.parse()?);
+            let contact_id_1: u32 = arg1.parse()?;
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
@@ -760,7 +758,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::set_chat_name(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                format!("{} {}", arg1, arg2).trim(),
+                &format!("{} {}", arg1, arg2).trim(),
             )
             .await?;
 
@@ -909,16 +907,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             Some(msg_id) => println!("sync message sent as {}.", msg_id),
             None => println!("sync message not needed."),
         },
-        "sendupdate" => {
-            ensure!(
-                !arg1.is_empty() && !arg2.is_empty(),
-                "Arguments <msg-id> <json status update> expected"
-            );
-            let msg_id = MsgId::new(arg1.parse()?);
-            context
-                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
-                .await?;
-        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -926,24 +914,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let query = format!("{} {}", arg1, arg2).trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, &query).await?;
+            let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!(
-                "{}{} messages for {}search of \"{}\"",
-                msglist.len(),
-                if msglist.len() == 1000 { "+" } else { "" },
-                if chat.is_none() {
-                    "global "
-                } else {
-                    "in-chat-"
-                },
-                query,
-            );
+            println!("{} messages.", msglist.len());
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1100,13 +1077,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
-        "resend" => {
-            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
-
-            let mut msg_ids = [MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            chat::resend_msgs(&context, &msg_ids).await?;
-        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
@@ -1146,7 +1116,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id = ContactId::new(arg1.parse()?);
+            let contact_id: u32 = arg1.parse()?;
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
@@ -1172,7 +1142,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -1181,16 +1151,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(&context, ContactId::new(arg1.parse()?)).await?;
+            Contact::delete(&context, arg1.parse()?).await?;
         }
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
+            let contact_id = arg1.parse()?;
             Contact::block(&context, contact_id).await?;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
+            let contact_id = arg1.parse()?;
             Contact::unblock(&context, contact_id).await?;
         }
         "listblocked" => {
@@ -1231,6 +1201,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
         }
+        // TODO: implement this again, unclear how to match this through though, without writing a parser.
+        // "event" => {
+        //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
+        //     let event = arg1.parse()?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
+        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
+        //     println!(
+        //         "Sending event {:?}({}), received value {}.",
+        //         event, event as usize, r,
+        //     );
+        // }
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 

examples/repl/main.rs

@@ -169,7 +169,7 @@ const DB_COMMANDS: [&str; 10] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 36] = [
+const CHAT_COMMANDS: [&str; 35] = [
     "listchats",
     "listarchived",
     "chat",
@@ -191,7 +191,6 @@ const CHAT_COMMANDS: [&str; 36] = [
     "sendfile",
     "sendhtml",
     "sendsyncmsg",
-    "sendupdate",
     "videochat",
     "draft",
     "listmedia",
@@ -207,12 +206,11 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 7] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
-    "resend",
     "markseen",
     "delmsg",
     "download",
@@ -228,12 +226,13 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 11] = [
+const MISC_COMMANDS: [&str; 12] = [
     "getqr",
     "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
+    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -416,7 +415,7 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
             let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
             if !qr.is_empty() {
                 if arg0 == "getbadqr" && qr.len() > 40 {
@@ -433,7 +432,7 @@ async fn handle_cmd(
         }
         "getqrsvg" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
             let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {

python/examples/test_examples.py

@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1, = acfactory.get_online_accounts(1)
+    ac1 = acfactory.get_one_online_account()
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -42,7 +42,7 @@ def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
     botproc = acfactory.run_bot_process(group_tracking, ffi=False)
 
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
         *ac_configure_completed*

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -1,8 +1,5 @@
 [mypy]
 
-[mypy-py.*]
-ignore_missing_imports = True
-
 [mypy-deltachat.capi.*]
 ignore_missing_imports = True
 
@@ -20,7 +17,3 @@ ignore_missing_imports = True
 
 [mypy-_pytest.*]
 ignore_missing_imports = True
-
-[mypy-imap_tools.*]
-ignore_missing_imports = True
-

python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

python/setup.py

@@ -11,11 +11,8 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
-        setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
-        ],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        setup_requires=['setuptools_scm'], # required for compatibility with `python3 setup.py sdist`
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ import sys
 
 from . import capi, const, hookspec # noqa
 from .capi import ffi  # noqa
-from .account import Account, get_core_info  # noqa
+from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa

python/src/deltachat/_build.py

@@ -8,9 +8,9 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import types
 
 import cffi
-import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,31 +19,36 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = {}
+    flags = types.SimpleNamespace()
     if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+    return flags
 
 
 def extract_functions(flags):
@@ -64,7 +69,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -100,7 +105,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
+                       include_dirs=flags.incs,
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -178,7 +183,10 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        **flags,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/account.py

@@ -22,29 +22,6 @@ class MissingCredentials(ValueError):
     """ Account is missing `addr` and `mail_pw` config values. """
 
 
-def get_core_info():
-    """ get some system info. """
-    from tempfile import NamedTemporaryFile
-
-    with NamedTemporaryFile() as path:
-        path.close()
-        return get_dc_info_as_dict(ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
-            lib.dc_context_unref,
-        ))
-
-
-def get_dc_info_as_dict(dc_context):
-    lines = from_dc_charpointer(lib.dc_get_info(dc_context))
-    info_dict = {}
-    for line in lines.split("\n"):
-        if not line.strip():
-            continue
-        key, value = line.split("=", 1)
-        info_dict[key.lower()] = value
-    return info_dict
-
-
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
@@ -90,9 +67,6 @@ class Account(object):
         """ re-enable logging. """
         self._logging = True
 
-    def __repr__(self):
-        return "<Account path={}>".format(self.db_path)
-
     # def __del__(self):
     #    self.shutdown()
 
@@ -107,7 +81,14 @@ class Account(object):
 
     def get_info(self) -> Dict[str, str]:
         """ return dictionary of built config parameters. """
-        return get_dc_info_as_dict(self._dc_context)
+        lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
+        d = {}
+        for line in lines.split("\n"):
+            if not line.strip():
+                continue
+            key, value = line.split("=", 1)
+            d[key.lower()] = value
+        return d
 
     def dump_account_info(self, logfile):
         def log(*args, **kwargs):
@@ -148,8 +129,6 @@ class Account(object):
         namebytes = name.encode("utf8")
         if namebytes == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
-        if isinstance(value, (int, bool)):
-            value = str(int(value))
         if value is not None:
             valuebytes = value.encode("utf8")
         else:
@@ -190,7 +169,7 @@ class Account(object):
         :returns: None
         """
         for key, value in kwargs.items():
-            self.set_config(key, value)
+            self.set_config(key, str(value))
 
     def is_configured(self) -> bool:
         """ determine if the account is configured already; an initial connection
@@ -200,12 +179,6 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def is_open(self) -> bool:
-        """Determine if account is open
-
-        :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
-
     def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
 
@@ -590,8 +563,6 @@ class Account(object):
         """ add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
-        if name and self._pm.has_plugin(name=name):
-            self._pm.unregister(name=name)
         self._pm.register(plugin, name=name)
         self._pm.check_pending()
         return plugin
@@ -695,25 +666,28 @@ class Account(object):
         if self._dc_context is None:
             return
 
-        # mark the event thread for shutdown (latest on next incoming event)
-        self._event_thread.mark_shutdown()
-
-        # stop_io also causes an info event which will wake up
-        # the EventThread's inner loop and let it notice the shutdown marker.
         self.stop_io()
 
+        self.log("remove dc_context references")
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
+        self._dc_context = None
+
         self.log("wait for event thread to finish")
         try:
-            self._event_thread.wait(timeout=5)
+            self._event_thread.wait(timeout=2)
         except RuntimeError as e:
             self.log("Waiting for event thread failed: {}".format(e))
 
         if self._event_thread.is_alive():
             self.log("WARN: event thread did not terminate yet, ignoring.")
 
-        self.log("remove dc_context references, making the Account unusable")
-        self._dc_context = None
-
         self._shutdown_event.set()
 
         hook = hookspec.Global._get_plugin_manager().hook

python/src/deltachat/chat.py

@@ -5,7 +5,7 @@ import calendar
 import json
 from datetime import datetime, timezone
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
@@ -517,7 +517,7 @@ class Chat(object):
                     lib.dc_array_get_timestamp(dc_array, i),
                     timezone.utc
                 ),
-                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/direct_imap.py

@@ -4,20 +4,63 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
+import email
 import ssl
 import pathlib
-from contextlib import contextmanager
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
 import imaplib
+import deltachat
 from deltachat import const, Account
-from typing import List
 
 
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
 class DirectImap:
     def __init__(self, account: Account) -> None:
         self.account = account
@@ -45,30 +88,37 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
         try:
             self.conn.logout()
-        except (OSError, imaplib.IMAP4.abort):
+        except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.folder.create(foldername)
-        except errors.MailboxFolderCreateError as e:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername: str) -> tuple:
+    def select_folder(self, foldername):
         assert not self._idling
-        return self.conn.folder.set(foldername)
+        return self.conn.select_folder(foldername)
 
-    def select_config_folder(self, config_name: str):
+    def select_config_folder(self, config_name):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -77,36 +127,50 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self) -> List[str]:
+    def list_folders(self):
         """ return list of all existing folder names"""
         assert not self._idling
-        return [folder.name for folder in self.conn.folder.list()]
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
 
-    def delete(self, uid_list: str, expunge=True):
+    def delete(self, range, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.set_flags(range, [DELETED])
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self) -> List[MailMessage]:
+    def get_all_messages(self):
         assert not self._idling
-        return [mail for mail in self.conn.fetch()]
 
-    def get_unread_messages(self) -> List[str]:
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
         assert not self._idling
-        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
+            res = self.conn.set_flags(messages, [SEEN])
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self) -> int:
+    def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -128,84 +192,75 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            for msg in self.conn.fetch(mark_seen=False):
-                body = getattr(msg.obj, "text", None)
-                if not body:
-                    body = getattr(msg.obj, "html", None)
-                if not body:
-                    log("Message", msg.uid, "has empty body")
+            requested = [b'BODY.PEEK[]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
                     continue
 
+                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(msg.uid))
-                fn.write_bytes(body)
-                log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
 
         print(stream.getvalue(), file=logfile)
 
-    @contextmanager
-    def idle(self):
-        """ return Idle ContextManager. """
-        idle_manager = IdleManager(self)
-        try:
-            yield idle_manager
-        finally:
-            idle_manager.done()
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
 
-    def append(self, folder: str, msg: str):
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res
+
+    def append(self, folder, msg):
         """Upload a message to *folder*.
         Trailing whitespace or a linebreak at the beginning will be removed automatically.
         """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
+        self.conn.append(folder, msg)
 
-    def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
+    def get_uid_by_message_id(self, message_id):
+        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]
-
-
-class IdleManager:
-    def __init__(self, direct_imap):
-        self.direct_imap = direct_imap
-        self.log = direct_imap.account.log
-        # fetch latest messages before starting idle so that it only
-        # returns messages that arrive anew
-        self.direct_imap.conn.fetch("1:*")
-        self.direct_imap.conn.idle.start()
-
-    def check(self, timeout=None) -> List[bytes]:
-        """ (blocking) wait for next idle message from server. """
-        self.log("imap-direct: calling idle_check")
-        res = self.direct_imap.conn.idle.poll(timeout=timeout)
-        self.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def wait_for_new_message(self, timeout=None) -> bytes:
-        while 1:
-            for item in self.check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
-                    return item
-
-    def wait_for_seen(self, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
-        """
-        while 1:
-            for item in self.check(timeout=timeout):
-                if FETCH in item:
-                    self.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        return int(item.split(b' ')[1])
-
-    def done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        res = self.direct_imap.conn.idle.stop()
-        return res

python/src/deltachat/events.py

@@ -1,8 +1,5 @@
 import threading
-import sys
-import traceback
 import time
-import io
 import re
 import os
 from queue import Queue, Empty
@@ -32,12 +29,10 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, init_time=None) -> None:
+    def __init__(self, account) -> None:
         self.account = account
-        self.logid = logid or self.account.get_config("displayname")
-        if init_time is None:
-            init_time = time.time()
-        self.init_time = init_time
+        self.logid = self.account.get_config("displayname")
+        self.init_time = time.time()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
@@ -161,14 +156,14 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_idle_inbox_ready(self):
+    def wait_all_initial_fetches(self):
         """Has to be called after start_io() to wait for fetch_existing_msgs to run
         so that new messages are not mistaken for old ones:
         - ac1 and ac2 are created
         - ac1 sends a message to ac2
         - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
         - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("INBOX: Idle entering")
+        self.get_info_contains("Done fetching existing messages")
 
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
@@ -230,7 +225,9 @@ class EventThread(threading.Thread):
         )
         while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL or self._marked_for_shutdown:
+            if event == ffi.NULL:
+                break
+            if self._marked_for_shutdown:
                 break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
@@ -244,23 +241,15 @@ class EventThread(threading.Thread):
 
             lib.dc_event_unref(event)
             ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            with self.swallow_and_log_exception("ac_process_ffi_event {}".format(ffi_event)):
+            try:
                 self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-            for name, kwargs in self._map_ffi_event(ffi_event):
-                hook = getattr(self.account._pm.hook, name)
-                info = "call {} kwargs={} failed".format(name, kwargs)
-                with self.swallow_and_log_exception(info):
+                for name, kwargs in self._map_ffi_event(ffi_event):
+                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
+                    hook = getattr(self.account._pm.hook, name)
                     hook(**kwargs)
-
-    @contextmanager
-    def swallow_and_log_exception(self, info):
-        try:
-            yield
-        except Exception as ex:
-            logfile = io.StringIO()
-            traceback.print_exception(*sys.exc_info(), file=logfile)
-            self.account.log("{}\nException {}\nTraceback:\n{}"
-                             .format(info, ex, logfile.getvalue()))
+            except Exception:
+                if self.account._dc_context is not None:
+                    raise
 
     def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name

python/src/deltachat/testplugin.py

@@ -8,43 +8,35 @@ import threading
 import fnmatch
 import time
 import weakref
-from queue import Queue
-from typing import List, Callable
+import tempfile
+from typing import List, Dict, Callable
 
 import pytest
 import requests
-import py
 
-from . import Account, const, account_hookimpl, get_core_info
+from . import Account, const
+from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
 
 def pytest_addoption(parser):
-    group = parser.getgroup("deltachat testplugin options")
-    group.addoption(
+    parser.addoption(
         "--liveconfig", action="store", default=None,
         help="a file with >=2 lines where each line "
              "contains NAME=VALUE config settings for one account"
     )
-    group.addoption(
+    parser.addoption(
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    group.addoption(
+    parser.addoption(
         "--strict-tls", action="store_true",
         help="Never accept invalid TLS certificates for test accounts",
     )
-    group.addoption(
-        "--extra-info", action="store_true",
-        help="show more info on failures (imap server state, config)"
-    )
-    group.addoption(
-        "--debug-setup", action="store_true",
-        help="show events during configure and start io phases of online accounts"
-    )
 
 
 def pytest_configure(config):
@@ -99,12 +91,9 @@ def pytest_configure(config):
 
         @pytest.hookimpl(hookwrapper=True)
         def pytest_runtest_teardown(self, item):
-            logging = item.config.getoption("--extra-info")
-            if logging:
-                self.enable_logging(item)
+            self.enable_logging(item)
             yield
-            if logging:
-                self.disable_logging(item)
+            self.disable_logging(item)
 
     la = LoggingAspect()
     config.pluginmanager.register(la)
@@ -112,12 +101,20 @@ def pytest_configure(config):
 
 
 def pytest_report_header(config, startdir):
-    info = get_core_info()
-    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
-        info['deltachat_core_version'],
-        info['sqlite_version'],
-        info['journal_mode'],
-    )]
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+         info['journal_mode'],
+     )])
 
     cfg = config.option.liveconfig
     if cfg:
@@ -129,99 +126,57 @@ def pytest_report_header(config, startdir):
     return summary
 
 
-@pytest.fixture(scope="session")
-def testprocess(request):
-    return TestProcess(pytestconfig=request.config)
+class SessionLiveConfigFromFile:
+    def __init__(self, fn) -> None:
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index: int):
+        return self.configlist[index]
+
+    def exists(self) -> bool:
+        return bool(self.configlist)
 
 
-class TestProcess:
-    """ A pytest session-scoped instance to help with managing "live" account configurations.
-    """
-    def __init__(self, pytestconfig):
-        self.pytestconfig = pytestconfig
-        self._addr2files = {}
-        self._configlist = []
+class SessionLiveConfigFromURL:
+    configlist: List[Dict[str, str]]
 
-    def get_liveconfig_producer(self):
-        """ provide live account configs, cached on a per-test-process scope
-        so that test functions can re-use already known live configs.
-        Depending on the --liveconfig option this comes from
-        a HTTP provider or a file with a line specifying each accounts config.
-        """
-        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
-        if not liveconfig_opt:
-            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
+    def __init__(self, url: str) -> None:
+        self.configlist = []
+        self.url = url
 
-        if not liveconfig_opt.startswith("http"):
-            for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith('#'):
-                    d = {}
-                    for part in line.split():
-                        name, value = part.split("=")
-                        d[name] = value
-                    self._configlist.append(d)
-
-            yield from iter(self._configlist)
-        else:
-            MAX_LIVE_CREATED_ACCOUNTS = 10
-            for index in range(MAX_LIVE_CREATED_ACCOUNTS):
-                try:
-                    yield self._configlist[index]
-                except IndexError:
-                    res = requests.post(liveconfig_opt)
-                    if res.status_code != 200:
-                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
-                                    index, res.status_code, res.text))
-                    d = res.json()
-                    config = dict(addr=d["email"], mail_pw=d["password"])
-                    print("newtmpuser {}: addr={}".format(index, config["addr"]))
-                    self._configlist.append(config)
-                    yield config
-            pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS))
-
-    def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
-        db_target_path = py.path.local(db_target_path)
-        assert not db_target_path.exists()
-
-        print("checking cache for", cache_addr)
+    def get(self, index: int):
         try:
-            filescache = self._addr2files[cache_addr]
-        except KeyError:
-            print("CACHE FAIL for", cache_addr)
-            return False
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self) -> bool:
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
         else:
-            print("CACHE HIT for", cache_addr)
-            targetdir = db_target_path.dirpath()
-            write_dict_to_dir(filescache, targetdir)
-            return True
-
-    def cache_maybe_store_configured_db_files(self, acc):
-        addr = acc.get_config("addr")
-        assert acc.is_configured()
-        # don't overwrite existing entries
-        if addr not in self._addr2files:
-            print("storing cache for", addr)
-            basedir = py.path.local(acc.get_blobdir()).dirpath()
-            self._addr2files[addr] = create_dict_from_files_in_path(basedir)
-            return True
-
-
-def create_dict_from_files_in_path(path):
-    base = py.path.local(path)
-    cachedict = {}
-    for path in base.visit(fil=py.path.local.isfile):
-        cachedict[path.relto(base)] = path.read_binary()
-    return cachedict
-
-
-def write_dict_to_dir(dic, target_dir):
-    assert dic
-    target_dir = py.path.local(target_dir)
-    for relpath, content in dic.items():
-        path = target_dir.join(relpath)
-        if not path.dirpath().exists():
-            path.dirpath().ensure(dir=1)
-        path.write_binary(content)
+            return SessionLiveConfigFromFile(liveconfig_opt)
 
 
 @pytest.fixture
@@ -254,408 +209,257 @@ def data(request):
     return Data()
 
 
-class ACSetup:
-    """ accounts setup helper to deal with multiple configure-process
-    and io & imap initialization phases. From tests, use the higher level
-    public ACFactory methods instead of its private helper class.
-    """
-    CONFIGURING = "CONFIGURING"
-    CONFIGURED = "CONFIGURED"
-    IDLEREADY = "IDLEREADY"
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
-    def __init__(self, testprocess, init_time):
-        self._configured_events = Queue()
-        self._account2state = {}
-        self._imap_cleaned = set()
-        self.testprocess = testprocess
-        self.init_time = init_time
+    class AccountMaker:
+        _finalizers: List[Callable[[], None]]
+        _accounts: List[Account]
 
-    def log(self, *args):
-        print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
-
-    def add_configured(self, account):
-        """ add an already configured account. """
-        assert account.is_configured()
-        self._account2state[account] = self.CONFIGURED
-        self.log("added already configured account", account, account.get_config("addr"))
-        return
-
-    def start_configure(self, account, reconfigure=False):
-        """ add an account and start its configure process. """
-
-        if reconfigure:
-            assert account.is_configured()
-
-        class PendingTracker:
-            @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
-
-        account.add_account_plugin(PendingTracker(), name="pending_tracker")
-        self._account2state[account] = self.CONFIGURING
-        account.configure(reconfigure=reconfigure)
-        self.log("started {}configure on".format("re-" if reconfigure else ""), account)
-
-    def wait_one_configured(self, account):
-        """ wait until this account has successfully configured. """
-        if self._account2state[account] == self.CONFIGURING:
-            while 1:
-                acc = self._pop_config_success()
-                if acc == account:
-                    break
-            self.init_imap(acc)
-            self.init_logging(acc)
-            acc._evtracker.consume_events()
-
-    def wait_all_configured(self):
-        """ Wait for all unconfigured accounts to become finished. """
-        while self.CONFIGURING in self._account2state.values():
-            self._pop_config_success()
-
-    def bring_online(self):
-        """ Wait for all accounts to become ready to receive messages.
-
-        This will initialize logging, start IO and the direct_imap attribute
-        for each account which either is CONFIGURED already or which is CONFIGURING
-        and successfully completing the configuration process.
-        """
-        print("wait_all_configured finds accounts=", self._account2state)
-        for acc, state in self._account2state.items():
-            if state == self.CONFIGURED:
-                self._onconfigure_start_io(acc)
-                self._account2state[acc] = self.IDLEREADY
-
-        while self.CONFIGURING in self._account2state.values():
-            acc = self._pop_config_success()
-            self._onconfigure_start_io(acc)
-            self._account2state[acc] = self.IDLEREADY
-        print("finished, account2state", self._account2state)
-
-    def _pop_config_success(self):
-        acc, success = self._configured_events.get()
-        if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
-        self._account2state[acc] = self.CONFIGURED
-        return acc
-
-    def _onconfigure_start_io(self, acc):
-        self.init_imap(acc)
-        self.init_logging(acc)
-        acc.start_io()
-        print(acc._logid, "waiting for inbox IDLE to become ready")
-        acc._evtracker.wait_idle_inbox_ready()
-        acc._evtracker.consume_events()
-        acc.log("inbox IDLE ready")
-
-    def init_logging(self, acc):
-        """ idempotent function for initializing logging (will replace existing logger). """
-        logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
-        acc.add_account_plugin(logger, name="logger-" + acc._logid)
-
-    def init_imap(self, acc):
-        """ initialize direct_imap and cleanup server state. """
-        from deltachat.direct_imap import DirectImap
-
-        assert acc.is_configured()
-        if not hasattr(acc, "direct_imap"):
-            acc.direct_imap = DirectImap(acc)
-        addr = acc.get_config("addr")
-        if addr not in self._imap_cleaned:
-            imap = acc.direct_imap
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete("1:*", expunge=True)
-                else:
-                    imap.conn.folder.delete(folder)
-            acc.log("imap cleaned for addr {}".format(addr))
-            self._imap_cleaned.add(addr)
-
-
-class ACFactory:
-    _finalizers: List[Callable[[], None]]
-    _accounts: List[Account]
-
-    def __init__(self, request, testprocess, tmpdir, data) -> None:
-        self.init_time = time.time()
-        self.tmpdir = tmpdir
-        self.pytestconfig = request.config
-        self.data = data
-        self.testprocess = testprocess
-        self._liveconfig_producer = testprocess.get_liveconfig_producer()
-
-        self._finalizers = []
-        self._accounts = []
-        self._acsetup = ACSetup(testprocess, self.init_time)
-        self._preconfigured_keys = ["alice", "bob", "charlie",
+        def __init__(self) -> None:
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self._accounts = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-        self.set_logging_default(False)
-        request.addfinalizer(self.finalize)
+            self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
-    def log(self, *args):
-        print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args)
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
 
-    def finalize(self):
-        while self._finalizers:
-            fin = self._finalizers.pop()
-            fin()
-
-        while self._accounts:
-            acc = self._accounts.pop()
-            if acc is not None:
-                imap = getattr(acc, "direct_imap", None)
-                if imap is not None:
-                    imap.shutdown()
-                    del acc.direct_imap
+            while self._accounts:
+                acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
-    def get_next_liveconfig(self):
-        """ Base function to get functional online configurations
-        where we can make valid SMTP and IMAP connections with.
-        """
-        configdict = next(self._liveconfig_producer).copy()
-        if "e2ee_enabled" not in configdict:
-            configdict["e2ee_enabled"] = "1"
-
-        if self.pytestconfig.getoption("--strict-tls"):
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-        assert "addr" in configdict and "mail_pw" in configdict
-        return configdict
-
-    def _get_cached_account_copy(self, addr):
-        if addr in self.testprocess._addr2files:
-            return self._getaccount(addr)
-
-    def get_unconfigured_account(self):
-        return self._getaccount()
-
-    def _getaccount(self, try_cache_addr=None):
-        logid = "ac{}".format(len(self._accounts) + 1)
-
-        # we need to use fixed database basename for maybe_cache_* functions to work
-        path = self.tmpdir.mkdir(logid).join("dc.db")
-        if try_cache_addr:
-            self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
-        ac = Account(path.strpath, logging=self._logging)
-        ac._logid = logid  # later instantiated FFIEventLogger needs this
-        ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-        if self.pytestconfig.getoption("--debug-setup"):
-            self._acsetup.init_logging(ac)
-        self._accounts.append(ac)
-        return ac
-
-    def set_logging_default(self, logging):
-        self._logging = bool(logging)
-
-    def remove_preconfigured_keys(self):
-        self._preconfigured_keys = []
-
-    def _preconfigure_key(self, account, addr):
-        # Only set a preconfigured key if we haven't used it yet for another account.
-        try:
-            keyname = self._preconfigured_keys.pop(0)
-        except IndexError:
-            pass
-        else:
-            fname_pub = self.data.read_path("key/{name}-public.asc".format(name=keyname))
-            fname_sec = self.data.read_path("key/{name}-secret.asc".format(name=keyname))
-            if fname_pub and fname_sec:
-                account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                return True
-            else:
-                print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-    def get_pseudo_configured_account(self):
-        # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
-        acname = ac._logid
-        addr = "{}@offline.org".format(acname)
-        ac.update_config(dict(
-            addr=addr, displayname=acname, mail_pw="123",
-            configured_addr=addr, configured_mail_pw="123",
-            configured="1",
-        ))
-        self._preconfigure_key(ac, addr)
-        self._acsetup.init_logging(ac)
-        return ac
-
-    # XXX deprecate the next function?
-    def new_online_configuring_account(self, cache=False, **kwargs):
-        configdict = self.get_next_liveconfig()
-        configdict.update(kwargs)
-        return self._setup_online_configuring_account(configdict, cache=cache)
-
-    def get_online_second_device(self, ac1, **kwargs):
-        ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
-        if ac2 is None:
-            # some tests setup the primary account without causing caching.
-            configdict = kwargs.copy()
-            configdict["addr"] = ac1.get_config("addr")
-            configdict["mail_pw"] = ac1.get_config("mail_pw")
-            ac2 = self._setup_online_configuring_account(configdict, cache=False)
-        elif kwargs:
-            ac2.update_config(kwargs)
-            self._acsetup.add_configured(ac2)
-
-        self.bring_accounts_online()
-        return ac2
-
-    def get_online_multidevice_setup(self, copied=True):
-        """ Provide two accounts. The second uses the same credentials
-        and if copied is True, also the same database and blobs.
-        You can use copy=False to get a typical configuration where
-        a user unsuspectingly sets up a second device and expects it to
-        "just work" not knowing that an export/import is required.
-        """
-        ac1, = self.get_online_accounts(1)
-        if copied:
-            ac2 = self._get_cached_account_copy(addr=ac1.get_config("addr"))
-            self._acsetup.add_configured(ac2)
-        else:
-            config2 = dict(addr=ac1.get_config("addr"), mail_pw=ac1.get_config("mail_pw"))
-            ac2 = self._setup_online_configuring_account(config2, cache=False)
-        self.bring_accounts_online()
-        return ac1, ac2
-
-    def get_online_devnull_email(self):
-        return self.get_next_liveconfig()["addr"]
-
-    def _setup_online_configuring_account(self, configdict, cache=False):
-        ac = self._get_cached_account_copy(configdict["addr"]) if cache else None
-        if ac is not None:
-            # make sure we consume a preconfig key, as if we had created a fresh account
-            self._preconfigured_keys.pop(0)
-            self._acsetup.add_configured(ac)
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path, logging=self._logging)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
+            if not quiet:
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
+            self._accounts.append(ac)
             return ac
-        ac = self.prepare_account_from_liveconfig(configdict)
-        self._acsetup.start_configure(ac)
-        return ac
 
-    def prepare_account_from_liveconfig(self, configdict):
-        ac = self.get_unconfigured_account()
-        assert "addr" in configdict and "mail_pw" in configdict, configdict
-        configdict.setdefault("bcc_self", False)
-        configdict.setdefault("mvbox_move", False)
-        configdict.setdefault("sentbox_watch", False)
-        ac.update_config(configdict)
-        self._preconfigure_key(ac, configdict["addr"])
-        return ac
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
 
-    def wait_configured(self, account):
-        """ Wait until the specified account has successfully completed configure. """
-        self._acsetup.wait_one_configured(account)
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
-    def bring_accounts_online(self):
-        print("bringing accounts online")
-        self._acsetup.bring_online()
-        print("all accounts online")
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
 
-    def get_online_configured_accounts(self, configlist):
-        accounts = [self.new_online_configuring_account(cache=True, **config)
-                    for config in configlist]
-        self._acsetup.wait_all_configured()
-        for acc in accounts:
-            self._acsetup.init_imap(acc)
-        return accounts
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
 
-    def force_reconfigure(self, account):
-        self._acsetup.start_configure(account, reconfigure=True)
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
 
-    def get_online_accounts(self, num):
-        """ Return a list of configured and started Accounts.
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
 
-        This function creates plain online accounts and fill
-        a testprocess-scoped cache and re-use these plain accounts
-        on the next test function.
-        """
-        accounts = [self.new_online_configuring_account(cache=True) for i in range(num)]
-        self.bring_accounts_online()
-        for acc in accounts:
-            self.testprocess.cache_maybe_store_configured_db_files(acc)
-        return accounts
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
-    def run_bot_process(self, module, ffi=True):
-        fn = module.__file__
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            return ac, dict(configdict)
 
-        bot_cfg = self.get_next_liveconfig()
-        bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
+        def get_online_configuring_account(self, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac._configtracker = ac.configure()
+            return ac
 
-        # Forget ac as it will be opened by the bot subprocess
-        # but keep something in the list to not confuse account generation
-        self._accounts[self._accounts.index(bot_ac)] = None
+        def get_one_online_account(self, pre_generated_key=True, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, move=move)
+            self.wait_configure_and_start_io([ac1])
+            return ac1
 
-        args = [
-            sys.executable,
-            "-u",
-            fn,
-            "--email", bot_cfg["addr"],
-            "--password", bot_cfg["mail_pw"],
-            bot_ac.db_path,
-        ]
-        if ffi:
-            args.insert(-1, "--show-ffi")
-        print("$", " ".join(args))
-        popen = subprocess.Popen(
-            args=args,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-            bufsize=0,                 # line buffering
-            close_fds=True,            # close all FDs other than 0/1/2
-            universal_newlines=True    # give back text
-        )
-        bot = BotProcess(popen, addr=bot_cfg["addr"])
-        self._finalizers.append(bot.kill)
-        return bot
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            self.wait_configure_and_start_io([ac1, ac2])
+            return ac1, ac2
 
-    def dump_imap_summary(self, logfile):
-        for ac in self._accounts:
-            ac.dump_account_info(logfile=logfile)
-            imap = getattr(ac, "direct_imap", None)
-            if imap is not None:
-                imap.dump_imap_structures(self.tmpdir, logfile=logfile)
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io(accounts)
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
 
-    def get_accepted_chat(self, ac1: Account, ac2):
-        ac2.create_chat(ac1)
-        return ac1.create_chat(ac2)
+        def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
 
-    def introduce_each_other(self, accounts, sending=True):
-        to_wait = []
-        for i, acc in enumerate(accounts):
-            for acc2 in accounts[i + 1:]:
-                chat = self.get_accepted_chat(acc, acc2)
-                if sending:
-                    chat.send_text("hi")
-                    to_wait.append(acc2)
-                    acc2.create_chat(acc).send_text("hi back")
-                    to_wait.append(acc)
-        for acc in to_wait:
-            acc.log("waiting for incoming message")
-            acc._evtracker.wait_next_incoming_message()
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
+            ac._configtracker = ac.configure()
+            return ac
 
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
+            started_accounts = []
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
 
-@pytest.fixture
-def acfactory(request, tmpdir, testprocess, data):
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data)
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
+
+        def run_bot_process(self, module, ffi=True):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
+            args = [
+                sys.executable,
+                "-u",
+                fn,
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=0,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
+
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1: Account, ac2: Account):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
     yield am
     if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testprocess.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
     stdout_queue: queue.Queue
 
-    def __init__(self, popen, addr) -> None:
+    def __init__(self, popen, bot_cfg) -> None:
         self.popen = popen
-        self.addr = addr
+        self.addr = bot_cfg["addr"]
 
         # we read stdout as quickly as we can in a thread and make
         # the (unicode) lines available for readers through a queue.
@@ -679,7 +483,7 @@ class BotProcess:
     def kill(self) -> None:
         self.popen.kill()
 
-    def wait(self, timeout=None) -> None:
+    def wait(self, timeout=30) -> None:
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -688,7 +492,7 @@ class BotProcess:
             print("+++FNMATCH:", next_pattern)
             ignored = []
             while 1:
-                line = self.stdout_queue.get()
+                line = self.stdout_queue.get(timeout=15)
                 if line is None:
                     if ignored:
                         print("BOT stdout terminated after these lines")

python/src/deltachat/tracker.py

@@ -90,11 +90,11 @@ class ConfigureTracker:
             if data1 is None or evdata == data1:
                 break
 
-    def wait_finish(self, timeout=None):
+    def wait_finish(self):
         """ wait until configure is completed.
 
         Raise Exception if Configure failed
         """
-        if not self._configure_events.get(timeout=timeout):
+        if not self._configure_events.get():
             content = "\n".join(map(str, self._ffi_events))
             raise ConfigureFailed(content)

python/tests/bench_test_setup.py

@@ -1,20 +0,0 @@
-"""
-run with:
-
-     pytest -vv --durations=10 bench_empty.py
-
-to see timings of test setups.
-"""
-
-import pytest
-
-BENCH_NUM = 3
-
-
-class TestEmpty:
-    def test_prepare_setup_measurings(self, acfactory):
-        acfactory.get_online_accounts(BENCH_NUM)
-
-    @pytest.mark.parametrize("num", range(0, BENCH_NUM + 1))
-    def test_setup_online_accounts(self, acfactory, num):
-        acfactory.get_online_accounts(num)

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3)
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account

python/tests/test_0_complex_or_slow.py

@@ -1,462 +0,0 @@
-import sys
-import pytest
-
-
-class TestGroupStressTests:
-    def test_group_many_members_add_leave_remove(self, acfactory, lp):
-        accounts = acfactory.get_online_accounts(5)
-        acfactory.introduce_each_other(accounts)
-        ac1, ac5 = accounts.pop(), accounts.pop()
-
-        lp.sec("ac1: creating group chat with 3 other members")
-        chat = ac1.create_group_chat("title1", contacts=accounts)
-
-        lp.sec("ac1: send message to new group chat")
-        msg1 = chat.send_text("hello")
-        assert msg1.is_encrypted()
-        gossiped_timestamp = chat.get_summary()["gossiped_timestamp"]
-        assert gossiped_timestamp > 0
-
-        assert chat.num_contacts() == 3 + 1
-
-        lp.sec("ac2: checking that the chat arrived correctly")
-        ac2 = accounts[0]
-        msg2 = ac2._evtracker.wait_next_incoming_message()
-        assert msg2.text == "hello"
-        print("chat is", msg2.chat)
-        assert msg2.chat.num_contacts() == 4
-
-        lp.sec("ac3: checking that 'ac4' is a known contact")
-        ac3 = accounts[1]
-        msg3 = ac3._evtracker.wait_next_incoming_message()
-        assert msg3.text == "hello"
-        ac3_contacts = ac3.get_contacts()
-        assert len(ac3_contacts) == 4
-        ac4_contacts = ac3.get_contacts(query=accounts[2].get_config("addr"))
-        assert len(ac4_contacts) == 1
-
-        lp.sec("ac2: removing one contact")
-        to_remove = ac2.create_contact(accounts[-1])
-        msg2.chat.remove_contact(to_remove)
-
-        lp.sec("ac1: receiving system message about contact removal")
-        sysmsg = ac1._evtracker.wait_next_incoming_message()
-        assert to_remove.addr in sysmsg.text
-        assert sysmsg.chat.num_contacts() == 3
-
-        # Receiving message about removed contact does not reset gossip
-        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
-
-        lp.sec("ac1: sending another message to the chat")
-        chat.send_text("hello2")
-        msg = ac2._evtracker.wait_next_incoming_message()
-        assert msg.text == "hello2"
-        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
-
-        lp.sec("ac1: adding fifth member to the chat")
-        chat.add_contact(ac5)
-        # Adding contact to chat resets gossiped_timestamp
-        assert chat.get_summary()["gossiped_timestamp"] >= gossiped_timestamp
-
-        lp.sec("ac2: receiving system message about contact addition")
-        sysmsg = ac2._evtracker.wait_next_incoming_message()
-        assert ac5.get_config("configured_addr") in sysmsg.text
-        assert sysmsg.chat.num_contacts() == 4
-
-        lp.sec("ac5: waiting for message about addition to the chat")
-        sysmsg = ac5._evtracker.wait_next_incoming_message()
-        msg = sysmsg.chat.send_text("hello!")
-        # Message should be encrypted because keys of other members are gossiped
-        assert msg.is_encrypted()
-
-    def test_synchronize_member_list_on_group_rejoin(self, acfactory, lp):
-        """
-        Test that user recreates group member list when it joins the group again.
-        ac1 creates a group with two other accounts: ac2 and ac3
-        Then it removes ac2, removes ac3 and adds ac2 back.
-        ac2 did not see that ac3 is removed, so it should rebuild member list from scratch.
-        """
-        lp.sec("setting up accounts, accepted with each other")
-        accounts = acfactory.get_online_accounts(3)
-        acfactory.introduce_each_other(accounts)
-        ac1, ac2, ac3 = accounts
-
-        lp.sec("ac1: creating group chat with 2 other members")
-        chat = ac1.create_group_chat("title1", contacts=[ac2, ac3])
-        assert not chat.is_promoted()
-
-        lp.sec("ac1: send message to new group chat")
-        msg = chat.send_text("hello")
-        assert chat.is_promoted() and msg.is_encrypted()
-
-        assert chat.num_contacts() == 3
-
-        lp.sec("checking that the chat arrived correctly")
-        for ac in accounts[1:]:
-            msg = ac._evtracker.wait_next_incoming_message()
-            assert msg.text == "hello"
-            print("chat is", msg.chat)
-            assert msg.chat.num_contacts() == 3
-
-        lp.sec("ac1: removing ac2")
-        chat.remove_contact(ac2)
-
-        lp.sec("ac2: wait for a message about removal from the chat")
-        msg = ac2._evtracker.wait_next_incoming_message()
-
-        lp.sec("ac1: removing ac3")
-        chat.remove_contact(ac3)
-
-        lp.sec("ac1: adding ac2 back")
-        # Group is promoted, message is sent automatically
-        assert chat.is_promoted()
-        chat.add_contact(ac2)
-
-        lp.sec("ac2: check that ac3 is removed")
-        msg = ac2._evtracker.wait_next_incoming_message()
-
-        assert chat.num_contacts() == 2
-        assert msg.chat.num_contacts() == 2
-        acfactory.dump_imap_summary(sys.stdout)
-
-
-def test_qr_verified_group_and_chatting(acfactory, lp):
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
-    chat1 = ac1.create_group_chat("hello", verified=True)
-    assert chat1.is_protected()
-    qr = chat1.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    chat2 = ac2.qr_join_chat(qr)
-    assert chat2.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("ac2: read member added message")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.is_encrypted()
-    assert "added" in msg.text.lower()
-
-    lp.sec("ac1: send message")
-    msg_out = chat1.send_text("hello")
-    assert msg_out.is_encrypted()
-
-    lp.sec("ac2: read message and check it's verified chat")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "hello"
-    assert msg.chat.is_protected()
-    assert msg.is_encrypted()
-
-    lp.sec("ac2: send message and let ac1 read it")
-    chat2.send_text("world")
-    msg = ac1._evtracker.wait_next_incoming_message()
-    assert msg.text == "world"
-    assert msg.is_encrypted()
-
-    lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
-    qr = ac1.get_setup_contact_qr()
-
-    lp.sec("ac3: start QR-code based setup contact protocol")
-    ch = ac3.qr_setup_contact(qr)
-    assert ch.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("ac1: add ac3 to verified group")
-    chat1.add_contact(ac3)
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.is_encrypted()
-    assert msg.is_system_message()
-    assert not msg.error
-
-    lp.sec("ac2: send message and let ac3 read it")
-    chat2.send_text("hi")
-    # Skip system message about added member
-    ac3._evtracker.wait_next_incoming_message()
-    msg = ac3._evtracker.wait_next_incoming_message()
-    assert msg.text == "hi"
-    assert msg.is_encrypted()
-
-
-@pytest.mark.parametrize("mvbox_move", [False, True])
-def test_fetch_existing(acfactory, lp, mvbox_move):
-    """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
-    This way, we can already offer them some email addresses they can write to.
-
-    Also, the newest existing emails from each folder are fetched during onboarding.
-
-    Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
-
-    def assert_folders_configured(ac):
-        """There was a bug that scan_folders() set the configured folders to None under some circumstances.
-        So, check that they are still configured:"""
-        assert ac.get_config("configured_sentbox_folder") == "Sent"
-        if mvbox_move:
-            assert ac.get_config("configured_mvbox_folder")
-
-    ac1, ac2 = acfactory.get_online_configured_accounts([dict(mvbox_move=mvbox_move), dict()])
-    ac1.direct_imap.create_folder("Sent")
-    ac1.set_config("sentbox_watch", "1")
-
-    # We need to reconfigure to find the new "Sent" folder.
-    # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
-    # would also find the "Sent" folder, but it would be too late:
-    # The sentbox thread, started by `start_io()`, would have seen that there is no
-    # ConfiguredSentboxFolder and do nothing.
-    acfactory.force_reconfigure(ac1)
-    acfactory.bring_accounts_online()
-
-    assert_folders_configured(ac1)
-
-    assert ac1.direct_imap.select_config_folder("mvbox" if mvbox_move else "inbox")
-    with ac1.direct_imap.idle() as idle1:
-        lp.sec("send out message with bcc to ourselves")
-        ac1.set_config("bcc_self", "1")
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-        chat.send_text("message text")
-        assert_folders_configured(ac1)
-
-        lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
-        assert idle1.wait_for_seen()
-    assert_folders_configured(ac1)
-
-    lp.sec("create a cloned ac1 and fetch contact history during configure")
-    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
-    assert_folders_configured(ac1_clone)
-
-    lp.sec("check that ac2 contact was fetchted during configure")
-    ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
-    ac2_addr = ac2.get_config("addr")
-    assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
-    assert_folders_configured(ac1_clone)
-
-    lp.sec("check that messages changed events arrive for the correct message")
-    msg = ac1_clone._evtracker.wait_next_messages_changed()
-    assert msg.text == "message text"
-    assert_folders_configured(ac1)
-    assert_folders_configured(ac1_clone)
-
-
-def test_fetch_existing_msgs_group_and_single(acfactory, lp):
-    """There was a bug concerning fetch-existing-msgs:
-
-    A sent a message to you, adding you to a group. This created a contact request.
-    You wrote a message to A, creating a chat.
-    ...but the group stayed blocked.
-    So, after fetch-existing-msgs you have one contact request and one chat with the same person.
-
-    See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    lp.sec("receive a message")
-    ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
-    ac1._evtracker.wait_next_incoming_message()
-
-    lp.sec("send out message with bcc to ourselves")
-    with ac1.direct_imap.idle() as idle1:
-        ac1.set_config("bcc_self", "1")
-        ac1_ac2_chat = ac1.create_chat(ac2)
-        ac1_ac2_chat.send_text("outgoing, encrypted direct message, creating a chat")
-        # wait until the bcc_self message arrives
-        assert idle1.wait_for_seen()
-
-    lp.sec("Clone online account and let it fetch the existing messages")
-    ac1_clone = acfactory.get_online_second_device(ac1, fetch_existing_msgs=1)
-
-    chats = ac1_clone.get_chats()
-    assert len(chats) == 4  # two newly created chats + self-chat + device-chat
-    group_chat = [c for c in chats if c.get_name() == "group name"][0]
-    assert group_chat.is_group()
-    private_chat, = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
-    assert not private_chat.is_group()
-
-    group_messages = group_chat.get_messages()
-    assert len(group_messages) == 1
-    assert group_messages[0].text == "incoming, unencrypted group message"
-    private_messages = private_chat.get_messages()
-    assert len(private_messages) == 1
-    assert private_messages[0].text.startswith("outgoing, encrypted")
-
-
-def test_undecipherable_group(acfactory, lp):
-    """Test how group messages that cannot be decrypted are
-    handled.
-
-    Group name is encrypted and plaintext subject is set to "..." in
-    this case, so we should assign the messages to existing chat
-    instead of creating a new one. Since there is no existing group
-    chat, the messages should be assigned to 1-1 chat with the sender
-    of the message.
-    """
-
-    lp.sec("creating and configuring three accounts")
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    acfactory.introduce_each_other([ac1, ac2, ac3])
-
-    lp.sec("ac3 reinstalls DC and generates a new key")
-    ac3.stop_io()
-    acfactory.remove_preconfigured_keys()
-    ac4 = acfactory.get_online_second_device(ac3)
-    # Create contacts to make sure incoming messages are not treated as contact requests
-
-    lp.sec("ac1: creating group chat with 2 other members")
-    chat = ac1.create_group_chat("title", contacts=[ac2, ac3])
-
-    lp.sec("ac1: send message to new group chat")
-    msg = chat.send_text("hello")
-
-    lp.sec("ac2: checking that the chat arrived correctly")
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "hello"
-    assert msg.is_encrypted(), "Message is not encrypted"
-
-    # ac4 cannot decrypt the message.
-    # Error message should be assigned to the chat with ac1.
-    lp.sec("ac4: checking that message is assigned to the sender chat")
-    error_msg = ac4._evtracker.wait_next_incoming_message()
-    assert error_msg.error  # There is an error decrypting the message
-    assert error_msg.chat == chat41
-
-    lp.sec("ac2: sending a reply to the chat")
-    msg.chat.send_text("reply")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.text == "reply"
-    assert reply.is_encrypted(), "Reply is not encrypted"
-
-    lp.sec("ac4: checking that reply is assigned to ac2 chat")
-    error_reply = ac4._evtracker.wait_next_incoming_message()
-    assert error_reply.error  # There is an error decrypting the message
-    assert error_reply.chat == chat42
-
-    # Test that ac4 replies to error messages don't appear in the
-    # group chat on ac1 and ac2.
-    lp.sec("ac4: replying to ac1 and ac2")
-
-    # Otherwise reply becomes a contact request.
-    chat41.send_text("I can't decrypt your message, ac1!")
-    chat42.send_text("I can't decrypt your message, ac2!")
-
-    msg = ac1._evtracker.wait_next_incoming_message()
-    assert msg.error is None
-    assert msg.text == "I can't decrypt your message, ac1!"
-    assert msg.is_encrypted(), "Message is not encrypted"
-    assert msg.chat == ac1.create_chat(ac3)
-
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.error is None
-    assert msg.text == "I can't decrypt your message, ac2!"
-    assert msg.is_encrypted(), "Message is not encrypted"
-    assert msg.chat == ac2.create_chat(ac4)
-
-
-def test_ephemeral_timer(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    lp.sec("ac1: create chat with ac2")
-    chat1 = ac1.create_chat(ac2)
-    chat2 = ac2.create_chat(ac1)
-
-    lp.sec("ac1: set ephemeral timer to 60")
-    chat1.set_ephemeral_timer(60)
-
-    lp.sec("ac1: check that ephemeral timer is set for chat")
-    assert chat1.get_ephemeral_timer() == 60
-    chat1_summary = chat1.get_summary()
-    assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
-
-    lp.sec("ac2: receive system message about ephemeral timer modification")
-    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    system_message1 = ac2._evtracker.wait_next_incoming_message()
-    assert chat2.get_ephemeral_timer() == 60
-    assert system_message1.is_system_message()
-
-    # Disabled until markers are implemented
-    # assert "Ephemeral timer: 60\n" in system_message1.get_message_info()
-
-    lp.sec("ac2: send message to ac1")
-    sent_message = chat2.send_text("message")
-    assert sent_message.ephemeral_timer == 60
-    assert "Ephemeral timer: 60\n" in sent_message.get_message_info()
-
-    # Timer is started immediately for sent messages
-    assert sent_message.ephemeral_timestamp is not None
-    assert "Expires: " in sent_message.get_message_info()
-
-    lp.sec("ac1: waiting for message from ac2")
-    text_message = ac1._evtracker.wait_next_incoming_message()
-    assert text_message.text == "message"
-    assert text_message.ephemeral_timer == 60
-    assert "Ephemeral timer: 60\n" in text_message.get_message_info()
-
-    # Timer should not start until message is displayed
-    assert text_message.ephemeral_timestamp is None
-    assert "Expires: " not in text_message.get_message_info()
-    text_message.mark_seen()
-    text_message = ac1.get_message_by_id(text_message.id)
-    assert text_message.ephemeral_timestamp is not None
-    assert "Expires: " in text_message.get_message_info()
-
-    lp.sec("ac2: set ephemeral timer to 0")
-    chat2.set_ephemeral_timer(0)
-    ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-
-    lp.sec("ac1: receive system message about ephemeral timer modification")
-    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    system_message2 = ac1._evtracker.wait_next_incoming_message()
-    assert system_message2.ephemeral_timer is None
-    assert "Ephemeral timer: " not in system_message2.get_message_info()
-    assert chat1.get_ephemeral_timer() == 0
-
-
-def test_multidevice_sync_seen(acfactory, lp):
-    """Test that message marked as seen on one device is marked as seen on another."""
-    ac1, ac1_clone = acfactory.get_online_multidevice_setup()
-    ac2, = acfactory.get_online_accounts(1)
-
-    ac1.set_config("bcc_self", "1")
-    ac1_clone.set_config("bcc_self", "1")
-
-    ac1_chat = ac1.create_chat(ac2)
-    ac1_clone_chat = ac1_clone.create_chat(ac2)
-    ac2_chat = ac2.create_chat(ac1)
-
-    lp.sec("Send a message from ac2 to ac1 and check that it's 'fresh'")
-    ac2_chat.send_text("Hi")
-    ac1_message = ac1._evtracker.wait_next_incoming_message()
-    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
-    assert ac1_chat.count_fresh_messages() == 1
-    assert ac1_clone_chat.count_fresh_messages() == 1
-    assert ac1_message.is_in_fresh
-    assert ac1_clone_message.is_in_fresh
-
-    lp.sec("ac1 marks message as seen on the first device")
-    ac1.mark_seen_messages([ac1_message])
-    assert ac1_message.is_in_seen
-
-    lp.sec("ac1 clone detects that message is marked as seen")
-    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-    assert ev.data1 == ac1_clone_chat.id
-    assert ac1_clone_message.is_in_seen
-
-    lp.sec("Send an ephemeral message from ac2 to ac1")
-    ac2_chat.set_ephemeral_timer(60)
-    ac1._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    ac1._evtracker.wait_next_incoming_message()
-    ac1_clone._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")
-    ac1_clone._evtracker.wait_next_incoming_message()
-
-    ac2_chat.send_text("Foobar")
-    ac1_message = ac1._evtracker.wait_next_incoming_message()
-    ac1_clone_message = ac1_clone._evtracker.wait_next_incoming_message()
-    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
-    assert "Expires: " not in ac1_clone_message.get_message_info()
-    assert "Ephemeral timer: 60\n" in ac1_message.get_message_info()
-    assert "Expires: " not in ac1_clone_message.get_message_info()
-
-    ac1.mark_seen_messages([ac1_message])
-    assert ac1_message.is_in_seen
-    assert "Expires: " in ac1_message.get_message_info()
-    ev = ac1_clone._evtracker.get_matching("DC_EVENT_MSGS_NOTICED")
-    assert ev.data1 == ac1_clone_chat.id
-    assert ac1_clone_message.is_in_seen
-    # Test that the timer is started on the second device after synchronizing the seen status.
-    assert "Expires: " in ac1_clone_message.get_message_info()

python/tests/test_3_offline.py

@@ -1,634 +0,0 @@
-from __future__ import print_function
-import pytest
-import os
-import time
-from deltachat import const, Account
-from deltachat.message import Message
-from deltachat.hookspec import account_hookimpl
-from deltachat.capi import ffi, lib
-from deltachat.cutil import iter_array
-from datetime import datetime, timedelta, timezone
-
-
-@pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by me",
-        ("removed", "tmp1@x.org", "me")),
-    ("Group left by some one (tmp1@x.org).",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Group left by tmp1@x.org.",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
-    ("Member nothing bla bla", None),
-    ("Another unknown system message", None),
-])
-def test_parse_system_add_remove(msgtext, res):
-    from deltachat.message import parse_system_add_remove
-
-    out = parse_system_add_remove(msgtext)
-    assert out == res
-
-
-class TestOfflineAccountBasic:
-    def test_wrong_db(self, tmpdir):
-        p = tmpdir.join("hello.db")
-        p.write("123")
-        account = Account(p.strpath)
-        assert not account.is_open()
-
-    def test_os_name(self, tmpdir):
-        p = tmpdir.join("hello.db")
-        # we can't easily test if os_name is used in X-Mailer
-        # outgoing messages without a full Online test
-        # but we at least check Account accepts the arg
-        ac1 = Account(p.strpath, os_name="solarpunk")
-        ac1.get_info()
-
-    def test_preconfigure_keypair(self, acfactory, data):
-        ac = acfactory.get_unconfigured_account()
-        alice_public = data.read_path("key/alice-public.asc")
-        alice_secret = data.read_path("key/alice-secret.asc")
-        assert alice_public and alice_secret
-        ac._preconfigure_keypair("alice@example.org", alice_public, alice_secret)
-
-    def test_getinfo(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        d = ac1.get_info()
-        assert d["arch"]
-        assert d["number_of_chats"] == "0"
-        assert d["bcc_self"] == "0"
-
-    def test_is_not_configured(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        assert not ac1.is_configured()
-        with pytest.raises(ValueError):
-            ac1.check_is_configured()
-
-    def test_wrong_config_keys(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        with pytest.raises(KeyError):
-            ac1.set_config("lqkwje", "value")
-        with pytest.raises(KeyError):
-            ac1.get_config("lqkwje")
-
-    def test_set_config_int_conversion(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        ac1.set_config("mvbox_move", False)
-        assert ac1.get_config("mvbox_move") == "0"
-        ac1.set_config("mvbox_move", True)
-        assert ac1.get_config("mvbox_move") == "1"
-        ac1.set_config("mvbox_move", 0)
-        assert ac1.get_config("mvbox_move") == "0"
-        ac1.set_config("mvbox_move", 1)
-        assert ac1.get_config("mvbox_move") == "1"
-
-    def test_update_config(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        ac1.update_config(dict(mvbox_move=False))
-        assert ac1.get_config("mvbox_move") == "0"
-
-    def test_has_savemime(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        assert "save_mime_headers" in ac1.get_config("sys.config_keys").split()
-
-    def test_has_bccself(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        assert "bcc_self" in ac1.get_config("sys.config_keys").split()
-        assert ac1.get_config("bcc_self") == "0"
-
-    def test_selfcontact_if_unconfigured(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        assert not ac1.get_self_contact().addr
-
-    def test_selfcontact_configured(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        me = ac1.get_self_contact()
-        assert me.display_name
-        assert me.addr
-
-    def test_get_config_fails(self, acfactory):
-        ac1 = acfactory.get_unconfigured_account()
-        with pytest.raises(KeyError):
-            ac1.get_config("123123")
-
-    def test_empty_group_bcc_self_enabled(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        ac1.set_config("bcc_self", "1")
-        chat = ac1.create_group_chat(name="group1")
-        msg = chat.send_text("msg1")
-        assert msg in chat.get_messages()
-
-    def test_empty_group_bcc_self_disabled(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        ac1.set_config("bcc_self", "0")
-        chat = ac1.create_group_chat(name="group1")
-        msg = chat.send_text("msg1")
-        assert msg in chat.get_messages()
-
-
-class TestOfflineContact:
-    def test_contact_attr(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some1@example.org", name="some1")
-        str(contact1)
-        repr(contact1)
-        assert contact1 == contact2
-        assert contact1.id
-        assert contact1.addr == "some1@example.org"
-        assert contact1.display_name == "some1"
-        assert not contact1.is_blocked()
-        assert not contact1.is_verified()
-
-    def test_get_blocked(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some2@example.org", name="some2")
-        ac1.create_contact("some3@example.org", name="some3")
-        assert ac1.get_blocked_contacts() == []
-        contact1.block()
-        assert ac1.get_blocked_contacts() == [contact1]
-        contact2.block()
-        blocked = ac1.get_blocked_contacts()
-        assert len(blocked) == 2 and contact1 in blocked and contact2 in blocked
-        contact2.unblock()
-        assert ac1.get_blocked_contacts() == [contact1]
-
-    def test_create_self_contact(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact(ac1.get_config("addr"))
-        assert contact1.id == 1
-
-    def test_get_contacts_and_delete(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contacts = ac1.get_contacts()
-        assert len(contacts) == 1
-        assert contact1 in contacts
-
-        assert not ac1.get_contacts(query="some2")
-        assert ac1.get_contacts(query="some1")
-        assert not ac1.get_contacts(only_verified=True)
-        assert len(ac1.get_contacts(with_self=True)) == 2
-
-        assert ac1.delete_contact(contact1)
-        assert contact1 not in ac1.get_contacts()
-
-    def test_get_contacts_and_delete_fails(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact("some1@example.com", name="some1")
-        msg = contact1.create_chat().send_text("one message")
-        assert not ac1.delete_contact(contact1)
-        assert not msg.filemime
-
-    def test_create_chat_flexibility(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        ac2 = acfactory.get_pseudo_configured_account()
-        chat1 = ac1.create_chat(ac2)
-        chat2 = ac1.create_chat(ac2.get_self_contact().addr)
-        assert chat1 == chat2
-        ac3 = acfactory.get_unconfigured_account()
-        with pytest.raises(ValueError):
-            ac1.create_chat(ac3)
-
-    def test_contact_rename(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact = ac1.create_contact("some1@example.com", name="some1")
-        chat = ac1.create_chat(contact)
-        assert chat.get_name() == "some1"
-        ac1.create_contact("some1@example.com", name="renamed")
-        ev = ac1._evtracker.get_matching("DC_EVENT_CHAT_MODIFIED")
-        assert ev.data1 == chat.id
-        assert chat.get_name() == "renamed"
-
-
-class TestOfflineChat:
-    @pytest.fixture
-    def ac1(self, acfactory):
-        return acfactory.get_pseudo_configured_account()
-
-    @pytest.fixture
-    def chat1(self, ac1):
-        return ac1.create_contact("some1@example.org", name="some1").create_chat()
-
-    def test_display(self, chat1):
-        str(chat1)
-        repr(chat1)
-
-    def test_is_group(self, chat1):
-        assert not chat1.is_group()
-
-    def test_chat_by_id(self, chat1):
-        chat2 = chat1.account.get_chat_by_id(chat1.id)
-        assert chat2 == chat1
-        with pytest.raises(ValueError):
-            chat1.account.get_chat_by_id(123123)
-
-    def test_chat_idempotent(self, chat1, ac1):
-        contact1 = chat1.get_contacts()[0]
-        chat2 = contact1.create_chat()
-        assert chat2.id == chat1.id
-        assert chat2.get_name() == chat1.get_name()
-        assert chat1 == chat2
-        assert not (chat1 != chat2)
-
-        for ichat in ac1.get_chats():
-            if ichat.id == chat1.id:
-                break
-        else:
-            pytest.fail("could not find chat")
-
-    def test_group_chat_add_second_account(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        ac2 = acfactory.get_pseudo_configured_account()
-        chat = ac1.create_group_chat(name="title1")
-        with pytest.raises(ValueError):
-            chat.add_contact(ac2.get_self_contact())
-        contact = chat.add_contact(ac2)
-        assert contact.addr == ac2.get_config("addr")
-        assert contact.name == ac2.get_config("displayname")
-        assert contact.account == ac1
-        chat.remove_contact(ac2)
-
-    def test_group_chat_creation(self, ac1):
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact2 = ac1.create_contact("some2@example.org", name="some2")
-        chat = ac1.create_group_chat(name="title1", contacts=[contact1, contact2])
-        assert chat.get_name() == "title1"
-        assert contact1 in chat.get_contacts()
-        assert contact2 in chat.get_contacts()
-        assert not chat.is_promoted()
-        chat.set_name("title2")
-        assert chat.get_name() == "title2"
-
-        d = chat.get_summary()
-        print(d)
-        assert d["id"] == chat.id
-        assert d["type"] == chat.get_type()
-        assert d["name"] == chat.get_name()
-        assert d["archived"] == chat.is_archived()
-        # assert d["param"] == chat.param
-        assert d["color"] == chat.get_color()
-        assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image()
-        assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
-
-    def test_group_chat_creation_with_translation(self, ac1):
-        ac1.set_stock_translation(const.DC_STR_MSGGRPNAME, "abc %1$s xyz %2$s")
-        ac1._evtracker.consume_events()
-        with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_FILE, "xyz %1$s")
-        ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        with pytest.raises(ValueError):
-            ac1.set_stock_translation(const.DC_STR_CONTACT_NOT_VERIFIED, "xyz %2$s")
-        ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        with pytest.raises(ValueError):
-            ac1.set_stock_translation(500, "xyz %1$s")
-        ac1._evtracker.get_matching("DC_EVENT_WARNING")
-        chat = ac1.create_group_chat(name="homework", contacts=[])
-        assert chat.get_name() == "homework"
-        chat.send_text("Now we have a group for homework")
-        assert chat.is_promoted()
-        chat.set_name("Homework")
-        assert chat.get_messages()[-1].text == "abc homework xyz Homework by me."
-
-    @pytest.mark.parametrize("verified", [True, False])
-    def test_group_chat_qr(self, acfactory, ac1, verified):
-        ac2 = acfactory.get_pseudo_configured_account()
-        chat = ac1.create_group_chat(name="title1", verified=verified)
-        assert chat.is_group()
-        qr = chat.get_join_qr()
-        assert ac2.check_qr(qr).is_ask_verifygroup
-
-    def test_removing_blocked_user_from_group(self, ac1, lp):
-        """
-        Test that blocked contact is not unblocked when removed from a group.
-        See https://github.com/deltachat/deltachat-core-rust/issues/2030
-        """
-        lp.sec("Create a group chat with a contact")
-        contact = ac1.create_contact("some1@example.org")
-        group = ac1.create_group_chat("title", contacts=[contact])
-        group.send_text("First group message")
-
-        lp.sec("ac1 blocks contact")
-        contact.block()
-        assert contact.is_blocked()
-
-        lp.sec("ac1 removes contact from their group")
-        group.remove_contact(contact)
-        assert contact.is_blocked()
-
-        lp.sec("ac1 adding blocked contact unblocks it")
-        group.add_contact(contact)
-        assert not contact.is_blocked()
-
-    def test_get_set_profile_image_simple(self, ac1, data):
-        chat = ac1.create_group_chat(name="title1")
-        p = data.get_path("d.png")
-        chat.set_profile_image(p)
-        p2 = chat.get_profile_image()
-        assert open(p, "rb").read() == open(p2, "rb").read()
-        chat.remove_profile_image()
-        assert chat.get_profile_image() is None
-
-    def test_mute(self, ac1):
-        chat = ac1.create_group_chat(name="title1")
-        assert not chat.is_muted()
-        assert chat.get_mute_duration() == 0
-        chat.mute()
-        assert chat.is_muted()
-        assert chat.get_mute_duration() == -1
-        chat.unmute()
-        assert not chat.is_muted()
-        chat.mute(50)
-        assert chat.is_muted()
-        assert chat.get_mute_duration() <= 50
-        with pytest.raises(ValueError):
-            chat.mute(-51)
-
-        # Regression test, this caused Rust panic previously
-        chat.mute(2**63 - 1)
-        assert chat.is_muted()
-        assert chat.get_mute_duration() == -1
-
-    def test_delete_and_send_fails(self, ac1, chat1):
-        chat1.delete()
-        ac1._evtracker.wait_next_messages_changed()
-        with pytest.raises(ValueError):
-            chat1.send_text("msg1")
-
-    def test_prepare_message_and_send(self, ac1, chat1):
-        msg = chat1.prepare_message(Message.new_empty(chat1.account, "text"))
-        msg.set_text("hello world")
-        assert msg.text == "hello world"
-        assert msg.id > 0
-        chat1.send_prepared(msg)
-        assert "Sent" in msg.get_message_info()
-        str(msg)
-        repr(msg)
-        assert msg == ac1.get_message_by_id(msg.id)
-
-    def test_prepare_file(self, ac1, chat1):
-        blobdir = ac1.get_blobdir()
-        p = os.path.join(blobdir, "somedata.txt")
-        with open(p, "w") as f:
-            f.write("some data")
-        message = chat1.prepare_message_file(p)
-        assert message.id > 0
-        message.set_text("hello world")
-        assert message.is_out_preparing()
-        assert message.text == "hello world"
-        chat1.send_prepared(message)
-        assert "Sent" in message.get_message_info()
-
-    def test_message_eq_contains(self, chat1):
-        msg = chat1.send_text("msg1")
-        assert msg in chat1.get_messages()
-        assert not (msg not in chat1.get_messages())
-        str(msg)
-        repr(msg)
-
-    def test_message_send_text(self, chat1):
-        msg = chat1.send_text("msg1")
-        assert msg
-        assert msg.is_text()
-        assert not msg.is_audio()
-        assert not msg.is_video()
-        assert not msg.is_gif()
-        assert not msg.is_file()
-        assert not msg.is_image()
-
-        assert not msg.is_in_fresh()
-        assert not msg.is_in_noticed()
-        assert not msg.is_in_seen()
-        assert msg.is_out_pending()
-        assert not msg.is_out_failed()
-        assert not msg.is_out_delivered()
-        assert not msg.is_out_mdn_received()
-
-    def test_message_image(self, chat1, data, lp):
-        with pytest.raises(ValueError):
-            chat1.send_image(path="notexists")
-        fn = data.get_path("d.png")
-        lp.sec("sending image")
-        chat1.account._evtracker.consume_events()
-        msg = chat1.send_image(fn)
-        chat1.account._evtracker.get_matching("DC_EVENT_NEW_BLOB_FILE")
-        assert msg.is_image()
-        assert msg
-        assert msg.id > 0
-        assert os.path.exists(msg.filename)
-        assert msg.filemime == "image/png"
-
-    @pytest.mark.parametrize("typein,typeout", [
-            (None, "application/octet-stream"),
-            ("text/plain", "text/plain"),
-            ("image/png", "image/png"),
-    ])
-    def test_message_file(self, ac1, chat1, data, lp, typein, typeout):
-        lp.sec("sending file")
-        fn = data.get_path("r.txt")
-        msg = chat1.send_file(fn, typein)
-        assert msg
-        assert msg.id > 0
-        assert msg.is_file()
-        assert os.path.exists(msg.filename)
-        assert msg.filename.endswith(msg.basename)
-        assert msg.filemime == typeout
-        msg2 = chat1.send_file(fn, typein)
-        assert msg2 != msg
-        assert msg2.filename != msg.filename
-
-    def test_create_contact(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        email = "hello <hello@example.org>"
-        contact1 = ac1.create_contact(email)
-        assert contact1.addr == "hello@example.org"
-        assert contact1.name == "hello"
-        contact1 = ac1.create_contact(email, name="world")
-        assert contact1.name == "world"
-        contact2 = ac1.create_contact("display1 <x@example.org>", "real")
-        assert contact2.name == "real"
-
-    def test_create_chat_simple(self, acfactory):
-        ac1 = acfactory.get_pseudo_configured_account()
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact1.create_chat().send_text("hello")
-
-    def test_chat_message_distinctions(self, ac1, chat1):
-        past1s = datetime.now(timezone.utc) - timedelta(seconds=1)
-        msg = chat1.send_text("msg1")
-        ts = msg.time_sent
-        assert msg.time_received is None
-        assert ts.strftime("Y")
-        assert past1s < ts
-        contact = msg.get_sender_contact()
-        assert contact == ac1.get_self_contact()
-
-    def test_set_config_after_configure_is_forbidden(self, ac1):
-        assert ac1.get_config("mail_pw")
-        assert ac1.is_configured()
-        with pytest.raises(ValueError):
-            ac1.set_config("addr", "123@example.org")
-
-    def test_import_export_one_contact(self, acfactory, tmpdir):
-        backupdir = tmpdir.mkdir("backup")
-        ac1 = acfactory.get_pseudo_configured_account()
-        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
-        # send a text message
-        msg = chat.send_text("msg1")
-        # send a binary file
-        bin = tmpdir.join("some.bin")
-        with bin.open("w") as f:
-            f.write("\00123" * 10000)
-        msg = chat.send_file(bin.strpath)
-        contact = msg.get_sender_contact()
-        assert contact == ac1.get_self_contact()
-        assert not backupdir.listdir()
-        ac1.stop_io()
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
-        ac2 = acfactory.get_unconfigured_account()
-        ac2.import_all(path)
-        contacts = ac2.get_contacts(query="some1")
-        assert len(contacts) == 1
-        contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
-        chat2 = contact2.create_chat()
-        messages = chat2.get_messages()
-        assert len(messages) == 2
-        assert messages[0].text == "msg1"
-        assert os.path.exists(messages[1].filename)
-
-    def test_set_get_draft(self, chat1):
-        msg = Message.new_empty(chat1.account, "text")
-        msg1 = chat1.prepare_message(msg)
-        msg1.set_text("hello")
-        chat1.set_draft(msg1)
-        msg1.set_text("obsolete")
-        msg2 = chat1.get_draft()
-        assert msg2.text == "hello"
-        chat1.set_draft(None)
-        assert chat1.get_draft() is None
-
-    def test_qr_setup_contact(self, acfactory, lp):
-        ac1 = acfactory.get_pseudo_configured_account()
-        ac2 = acfactory.get_pseudo_configured_account()
-        qr = ac1.get_setup_contact_qr()
-        assert qr.startswith("OPENPGP4FPR:")
-        res = ac2.check_qr(qr)
-        assert res.is_ask_verifycontact()
-        assert not res.is_ask_verifygroup()
-        assert res.contact_id == 10
-
-    def test_quote(self, chat1):
-        """Offline quoting test"""
-        msg = Message.new_empty(chat1.account, "text")
-        msg.set_text("Multi\nline\nmessage")
-        assert msg.quoted_text is None
-
-        # Prepare message to assign it a Message-Id.
-        # Messages without Message-Id cannot be quoted.
-        msg = chat1.prepare_message(msg)
-
-        reply_msg = Message.new_empty(chat1.account, "text")
-        reply_msg.set_text("reply")
-        reply_msg.quote = msg
-        assert reply_msg.quoted_text == "Multi\nline\nmessage"
-
-    def test_group_chat_many_members_add_remove(self, ac1, lp):
-        lp.sec("ac1: creating group chat with 10 other members")
-        chat = ac1.create_group_chat(name="title1")
-        # promote chat
-        chat.send_text("hello")
-        assert chat.is_promoted()
-
-        # activate local plugin
-        in_list = []
-
-        class InPlugin:
-            @account_hookimpl
-            def ac_member_added(self, chat, contact, actor):
-                in_list.append(("added", chat, contact, actor))
-
-            @account_hookimpl
-            def ac_member_removed(self, chat, contact, actor):
-                in_list.append(("removed", chat, contact, actor))
-
-        ac1.add_account_plugin(InPlugin())
-
-        # perform add contact many times
-        contacts = []
-        for i in range(10):
-            lp.sec("create contact")
-            contact = ac1.create_contact("some{}@example.org".format(i))
-            contacts.append(contact)
-            lp.sec("add contact")
-            chat.add_contact(contact)
-
-        assert chat.num_contacts() == 11
-
-        # let's make sure the events perform plugin hooks
-        def wait_events(cond):
-            now = time.time()
-            while time.time() < now + 5:
-                if cond():
-                    break
-                time.sleep(0.1)
-            else:
-                pytest.fail("failed to get events")
-
-        wait_events(lambda: len(in_list) == 10)
-
-        assert len(in_list) == 10
-        chat_contacts = chat.get_contacts()
-        for in_cmd, in_chat, in_contact, in_actor in in_list:
-            assert in_cmd == "added"
-            assert in_chat == chat
-            assert in_contact in chat_contacts
-            assert in_actor is None
-            chat_contacts.remove(in_contact)
-
-        assert chat_contacts[0].id == 1  # self contact
-
-        in_list[:] = []
-
-        lp.sec("ac1: removing two contacts and checking things are right")
-        chat.remove_contact(contacts[9])
-        chat.remove_contact(contacts[3])
-        assert chat.num_contacts() == 9
-
-        wait_events(lambda: len(in_list) == 2)
-        assert len(in_list) == 2
-        assert in_list[0][0] == "removed"
-        assert in_list[0][1] == chat
-        assert in_list[0][2] == contacts[9]
-        assert in_list[1][0] == "removed"
-        assert in_list[1][1] == chat
-        assert in_list[1][2] == contacts[3]
-
-    def test_audit_log_view_without_daymarker(self, ac1, lp):
-        lp.sec("ac1: test audit log (show only system messages)")
-        chat = ac1.create_group_chat(name="audit log sample data")
-        # promote chat
-        chat.send_text("hello")
-        assert chat.is_promoted()
-
-        lp.sec("create test data")
-        chat.add_contact(ac1.create_contact("some-1@example.org"))
-        chat.set_name("audit log test group")
-        chat.send_text("a message in between")
-
-        lp.sec("check message count of all messages")
-        assert len(chat.get_messages()) == 4
-
-        lp.sec("check message count of only system messages (without daymarkers)")
-        dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
-            lib.dc_array_unref
-        )
-        assert len(list(iter_array(dc_array, lambda x: x))) == 2

python/tests/test_4_lowlevel.py

@@ -1,199 +0,0 @@
-
-import os
-
-from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
-from deltachat.capi import ffi
-from deltachat.capi import lib
-from deltachat.testplugin import ACSetup
-# from deltachat.account import EventLogger
-
-
-class TestACSetup:
-    def test_basic_states(self, acfactory, monkeypatch, testprocess):
-        pc = ACSetup(init_time=0.0, testprocess=testprocess)
-        acc = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
-        pc.start_configure(acc)
-        assert pc._account2state[acc] == pc.CONFIGURING
-        pc._configured_events.put((acc, True))
-        monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
-        pc.wait_one_configured(acc)
-        assert pc._account2state[acc] == pc.CONFIGURED
-        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
-        pc.bring_online()
-        assert pc._account2state[acc] == pc.IDLEREADY
-
-    def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory, testprocess):
-        pc = ACSetup(init_time=0.0, testprocess=testprocess)
-        monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
-        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
-        ac1 = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(ac1, "configure", lambda **kwargs: None)
-        pc.start_configure(ac1)
-        ac2 = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(ac2, "configure", lambda **kwargs: None)
-        pc.start_configure(ac2)
-        assert pc._account2state[ac1] == pc.CONFIGURING
-        assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac1, True))
-        pc.wait_one_configured(ac1)
-        assert pc._account2state[ac1] == pc.CONFIGURED
-        assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac2, True))
-        pc.bring_online()
-        assert pc._account2state[ac1] == pc.IDLEREADY
-        assert pc._account2state[ac2] == pc.IDLEREADY
-
-    def test_store_and_retrieve_configured_account_cache(self, acfactory, tmpdir):
-        ac1 = acfactory.get_pseudo_configured_account()
-        holder = acfactory._acsetup.testprocess
-        assert holder.cache_maybe_store_configured_db_files(ac1)
-        assert not holder.cache_maybe_store_configured_db_files(ac1)
-        acdir = tmpdir.mkdir("newaccount")
-        addr = ac1.get_config("addr")
-        target_db_path = acdir.join("db").strpath
-        assert holder.cache_maybe_retrieve_configured_db_files(addr, target_db_path)
-        assert len(os.listdir(acdir)) >= 2
-
-
-def test_liveconfig_caching(acfactory, monkeypatch):
-    prod = [
-        {"addr": "1@example.org", "mail_pw": "123"},
-    ]
-    acfactory._liveconfig_producer = iter(prod)
-    d1 = acfactory.get_next_liveconfig()
-    d1["hello"] = "world"
-    acfactory._liveconfig_producer = iter(prod)
-    d2 = acfactory.get_next_liveconfig()
-    assert "hello" not in d2
-
-
-def test_empty_context():
-    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_context_unref(ctx)
-
-
-def test_dc_close_events(tmpdir, acfactory):
-    ac1 = acfactory.get_unconfigured_account()
-
-    # register after_shutdown function
-    shutdowns = Queue()
-
-    class ShutdownPlugin:
-        @global_hookimpl
-        def dc_account_after_shutdown(self, account):
-            assert account._dc_context is None
-            shutdowns.put(account)
-    register_global_plugin(ShutdownPlugin())
-    assert hasattr(ac1, "_dc_context")
-    ac1.shutdown()
-    shutdowns.get(timeout=2)
-
-
-def test_wrong_db(tmpdir):
-    p = tmpdir.join("hello.db")
-    # write an invalid database file
-    p.write("x123" * 10)
-
-    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
-    assert not lib.dc_context_is_open(context)
-
-
-def test_empty_blobdir(tmpdir):
-    db_fname = tmpdir.join("hello.db")
-    # Apparently some client code expects this to be the same as passing NULL.
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
-        lib.dc_context_unref,
-    )
-    assert ctx != ffi.NULL
-
-
-def test_event_defines():
-    assert const.DC_EVENT_INFO == 100
-    assert const.DC_CONTACT_ID_SELF
-
-
-def test_sig():
-    sig = capi.lib.dc_event_has_string_data
-    assert not sig(const.DC_EVENT_MSGS_CHANGED)
-    assert sig(const.DC_EVENT_INFO)
-    assert sig(const.DC_EVENT_WARNING)
-    assert sig(const.DC_EVENT_ERROR)
-    assert sig(const.DC_EVENT_SMTP_CONNECTED)
-    assert sig(const.DC_EVENT_IMAP_CONNECTED)
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
-    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
-
-
-def test_markseen_invalid_message_ids(acfactory):
-    ac1 = acfactory.get_pseudo_configured_account()
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
-    chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    msg_ids = [9]
-    lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
-
-
-def test_get_special_message_id_returns_empty_message(acfactory):
-    ac1 = acfactory.get_pseudo_configured_account()
-    for i in range(1, 10):
-        msg = ac1.get_message_by_id(i)
-        assert msg.id == 0
-
-
-def test_provider_info_none():
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
-
-
-def test_get_info_open(tmpdir):
-    db_fname = tmpdir.join("test.db")
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
-        lib.dc_context_unref,
-    )
-    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' in info
-
-
-def test_logged_hook_failure(acfactory):
-    ac1 = acfactory.get_pseudo_configured_account()
-    cap = []
-    ac1.log = cap.append
-    with ac1._event_thread.swallow_and_log_exception("some"):
-        0/0
-    assert cap
-    assert "some" in str(cap)
-    assert "ZeroDivisionError" in str(cap)
-    assert "Traceback" in str(cap)
-
-
-def test_logged_ac_process_ffi_failure(acfactory):
-    from deltachat import account_hookimpl
-
-    ac1 = acfactory.get_pseudo_configured_account()
-
-    class FailPlugin:
-        @account_hookimpl
-        def ac_process_ffi_event(ffi_event):
-            0/0
-
-    cap = Queue()
-    ac1.log = cap.put
-    ac1.add_account_plugin(FailPlugin())
-    # cause any event eg contact added/changed
-    ac1.create_contact("something@example.org")
-    res = cap.get(timeout=10)
-    assert "ac_process_ffi_event" in res
-    assert "ZeroDivisionError" in res
-    assert "Traceback" in res

python/tests/test_increation.py

@@ -33,7 +33,7 @@ def wait_msgs_changed(account, msgs_list):
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
@@ -43,7 +43,7 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
@@ -56,7 +56,7 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
 
         chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?

python/tests/test_lowlevel.py

@@ -0,0 +1,104 @@
+from __future__ import print_function
+
+from queue import Queue
+from deltachat import capi, cutil, const
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
+from deltachat.capi import ffi
+from deltachat.capi import lib
+# from deltachat.account import EventLogger
+
+
+def test_empty_context():
+    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
+    capi.lib.dc_context_unref(ctx)
+
+
+def test_dc_close_events(tmpdir, acfactory):
+    ac1 = acfactory.get_unconfigured_account()
+
+    # register after_shutdown function
+    shutdowns = Queue()
+
+    class ShutdownPlugin:
+        @global_hookimpl
+        def dc_account_after_shutdown(self, account):
+            assert account._dc_context is None
+            shutdowns.put(account)
+    register_global_plugin(ShutdownPlugin())
+    assert hasattr(ac1, "_dc_context")
+    ac1.shutdown()
+    shutdowns.get(timeout=2)
+
+
+def test_wrong_db(tmpdir):
+    p = tmpdir.join("hello.db")
+    # write an invalid database file
+    p.write("x123" * 10)
+
+    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+
+
+def test_empty_blobdir(tmpdir):
+    db_fname = tmpdir.join("hello.db")
+    # Apparently some client code expects this to be the same as passing NULL.
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
+        lib.dc_context_unref,
+    )
+    assert ctx != ffi.NULL
+
+
+def test_event_defines():
+    assert const.DC_EVENT_INFO == 100
+    assert const.DC_CONTACT_ID_SELF
+
+
+def test_sig():
+    sig = capi.lib.dc_event_has_string_data
+    assert not sig(const.DC_EVENT_MSGS_CHANGED)
+    assert sig(const.DC_EVENT_INFO)
+    assert sig(const.DC_EVENT_WARNING)
+    assert sig(const.DC_EVENT_ERROR)
+    assert sig(const.DC_EVENT_SMTP_CONNECTED)
+    assert sig(const.DC_EVENT_IMAP_CONNECTED)
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
+    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
+
+
+def test_markseen_invalid_message_ids(acfactory):
+    ac1 = acfactory.get_configured_offline_account()
+
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
+    chat.send_text("one messae")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    msg_ids = [9]
+    lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+
+
+def test_get_special_message_id_returns_empty_message(acfactory):
+    ac1 = acfactory.get_configured_offline_account()
+    for i in range(1, 10):
+        msg = ac1.get_message_by_id(i)
+        assert msg.id == 0
+
+
+def test_provider_info_none():
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+
+
+def test_get_info_open(tmpdir):
+    db_fname = tmpdir.join("test.db")
+    ctx = ffi.gc(
+        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
+        lib.dc_context_unref,
+    )
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
+    assert 'database_dir' in info

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -78,8 +78,8 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 150
-timeout_func_only = True
+timeout = 90
+timeout_method = thread
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.
 

rust-toolchain

@@ -1 +1 @@
-1.60.0
+1.54.0

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.60.0
+RUST_VERSION=1.54.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_rust.sh

@@ -8,7 +8,7 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.60.0
+RUST_VERSION=1.54.0
 
 curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
 cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

src/accounts.rs

@@ -54,19 +54,10 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(
-            config_file.exists().await,
-            "{:?} does not exist",
-            config_file
-        );
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
 
-        let config = Config::from_file(config_file)
-            .await
-            .context("failed to load accounts config")?;
-        let accounts = config
-            .load_accounts()
-            .await
-            .context("failed to load accounts")?;
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
 
         let emitter = EventEmitter::new();
 
@@ -75,9 +66,7 @@ impl Accounts {
         emitter.sender.send(events.get_emitter()).await?;
 
         for account in accounts.values() {
-            emitter.add_account(account).await.with_context(|| {
-                format!("failed to add account {} to event emitter", account.id)
-            })?;
+            emitter.add_account(account).await?;
         }
 
         Ok(Self {
@@ -115,9 +104,7 @@ impl Accounts {
         Ok(())
     }
 
-    /// Add a new account and opens it.
-    ///
-    /// Returns account ID.
+    /// Add a new account.
     pub async fn add_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account(&self.dir).await?;
 
@@ -128,17 +115,6 @@ impl Accounts {
         Ok(account_config.id)
     }
 
-    /// Adds a new closed account.
-    pub async fn add_closed_account(&mut self) -> Result<u32> {
-        let account_config = self.config.new_account(&self.dir).await?;
-
-        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
-        self.accounts.insert(account_config.id, ctx);
-
-        Ok(account_config.id)
-    }
-
     /// Remove an account.
     pub async fn remove_account(&mut self, id: u32) -> Result<()> {
         let ctx = self.accounts.remove(&id);
@@ -148,27 +124,9 @@ impl Accounts {
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            // Spend up to 1 minute trying to remove the files.
-            // Files may remain locked up to 30 seconds due to r2d2 bug:
-            // https://github.com/sfackler/r2d2/issues/99
-            let mut counter = 0;
-            loop {
-                counter += 1;
-
-                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                    .await
-                    .context("failed to remove account data")
-                {
-                    if counter > 60 {
-                        return Err(err);
-                    }
-
-                    // Wait 1 second and try again.
-                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
-                } else {
-                    break;
-                }
-            }
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
         }
         self.config.remove_account(id).await?;
 
@@ -224,7 +182,7 @@ impl Accounts {
 
         match res {
             Ok(_) => {
-                let ctx = Context::new(new_dbfile, account_config.id).await?;
+                let ctx = Context::with_blobdir(new_dbfile, new_blobdir, account_config.id).await?;
                 self.emitter.add_account(&ctx).await?;
                 self.accounts.insert(account_config.id, ctx);
                 Ok(account_config.id)
@@ -425,15 +383,7 @@ impl Config {
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
-                .await
-                .with_context(|| {
-                    format!(
-                        "failed to create context from file {:?}",
-                        account_config.dbfile()
-                    )
-                })?;
-
+            let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
             accounts.insert(account_config.id, ctx);
         }
 
@@ -458,13 +408,8 @@ impl Config {
 
         self.sync().await?;
 
-        self.select_account(id)
-            .await
-            .context("failed to select just added account")?;
-        let cfg = self
-            .get_account(id)
-            .await
-            .context("failed to get just added account")?;
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
         Ok(cfg)
     }
 
@@ -756,49 +701,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_encrypted_account() -> Result<()> {
-        let dir = tempfile::tempdir().context("failed to create tempdir")?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create accounts manager")?;
-
-        assert_eq!(accounts.accounts.len(), 0);
-        let account_id = accounts
-            .add_closed_account()
-            .await
-            .context("failed to add closed account")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.id, account_id);
-        let passphrase_set_success = account
-            .open("foobar".to_string())
-            .await
-            .context("failed to set passphrase")?;
-        assert!(passphrase_set_success);
-        drop(accounts);
-
-        let accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create second accounts manager")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.is_open().await, false);
-
-        // Try wrong passphrase.
-        assert_eq!(account.open("barfoo".to_string()).await?, false);
-        assert_eq!(account.open("".to_string()).await?, false);
-
-        assert_eq!(account.open("foobar".to_string()).await?, true);
-        assert_eq!(account.is_open().await, true);
-
-        Ok(())
-    }
 }

src/aheader.rs

@@ -2,7 +2,7 @@
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
-use anyhow::{bail, Context as _, Error, Result};
+use anyhow::{bail, format_err, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
@@ -139,14 +139,15 @@ impl str::FromStr for Aheader {
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .context("keydata attribute is not found")
+            .ok_or_else(|| format_err!("keydata attribute is not found"))
             .and_then(|raw| {
-                SignedPublicKey::from_base64(&raw).context("autocrypt key cannot be decoded")
+                SignedPublicKey::from_base64(&raw)
+                    .map_err(|_| format_err!("Autocrypt key cannot be decoded"))
             })
             .and_then(|key| {
                 key.verify()
                     .and(Ok(key))
-                    .context("autocrypt key cannot be verified")
+                    .map_err(|_| format_err!("Autocrypt key cannot be verified"))
             })?;
 
         let prefer_encrypt = attributes

src/blob.rs

@@ -3,26 +3,28 @@
 use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
-use std::io::Cursor;
 
 use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::{format_err, Context as _, Error};
-use image::{DynamicImage, ImageFormat};
+use anyhow::format_err;
+use anyhow::Context as _;
+use anyhow::Error;
+use image::DynamicImage;
+use image::GenericImageView;
+use image::ImageFormat;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::{
-    MediaQuality, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE, WORSE_IMAGE_SIZE,
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
 };
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::LogExt;
 use crate::message;
-use crate::message::Viewtype;
 
 /// Represents a file in the blob directory.
 ///
@@ -61,7 +63,7 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -85,16 +87,13 @@ impl<'a> BlobObject<'a> {
 
     // Creates a new file, returning a tuple of the name and the handle.
     async fn create_new_file(
-        context: &Context,
         dir: &Path,
         stem: &str,
         ext: &str,
     ) -> Result<(String, fs::File), BlobError> {
-        const MAX_ATTEMPT: u32 = 16;
-        let mut attempt = 0;
+        let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
-        loop {
-            attempt += 1;
+        for attempt in 0..max_attempt {
             let path = dir.join(&name);
             match fs::OpenOptions::new()
                 .create_new(true)
@@ -104,20 +103,24 @@ impl<'a> BlobObject<'a> {
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
-                    if attempt >= MAX_ATTEMPT {
+                    if attempt == max_attempt {
                         return Err(BlobError::CreateFailure {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
                         });
-                    } else if attempt == 1 && !dir.exists().await {
-                        fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
                     }
                 }
             }
         }
+        // This is supposed to be unreachable, but the compiler doesn't know.
+        Err(BlobError::CreateFailure {
+            blobdir: dir.to_path_buf(),
+            blobname: name,
+            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+        })
     }
 
     /// Creates a new blob object with unique name by copying an existing file.
@@ -146,7 +149,7 @@ impl<'a> BlobObject<'a> {
             })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
-            BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
+            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
@@ -289,7 +292,7 @@ impl<'a> BlobObject<'a> {
 
     /// Returns the filename of the blob.
     pub fn as_file_name(&self) -> &str {
-        self.name.rsplit('/').next().unwrap()
+        self.name.rsplitn(2, '/').next().unwrap()
     }
 
     /// The path relative in the blob directory.
@@ -302,7 +305,7 @@ impl<'a> BlobObject<'a> {
     /// If a blob's filename has an extension, it is always guaranteed
     /// to be lowercase.
     pub fn suffix(&self) -> Option<&str> {
-        let ext = self.name.rsplit('.').next();
+        let ext = self.name.rsplitn(2, '.').next();
         if ext == Some(&self.name) {
             None
         } else {
@@ -345,30 +348,13 @@ impl<'a> BlobObject<'a> {
         };
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
-        // Let's take the tricky filename
-        // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
-        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
         let mut iter = clean.splitn(2, '.');
-
         let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        // stem == "file"
-
-        let ext_chars = iter.next().unwrap_or_default().chars();
-        let ext: String = ext_chars
-            .rev()
-            .take(32)
-            .collect::<Vec<_>>()
-            .iter()
-            .rev()
-            .collect();
-        // ext == "d_point_and_double_ending.tar.gz"
-
+        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
         if ext.is_empty() {
             (stem, "".to_string())
         } else {
             (stem, format!(".{}", ext).to_lowercase())
-            // Return ("file", ".d_point_and_double_ending.tar.gz")
-            // which is not perfect but acceptable.
         }
     }
 
@@ -463,8 +449,7 @@ impl<'a> BlobObject<'a> {
 
         fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
             encoded.clear();
-            let mut buf = Cursor::new(encoded);
-            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
+            img.write_to(encoded, image::ImageFormat::Jpeg)?;
             Ok(())
         }
         fn encoded_img_exceeds_bytes(
@@ -634,15 +619,17 @@ pub enum BlobError {
 mod tests {
     use fs::File;
 
-    use anyhow::Result;
-    use image::{GenericImageView, Pixel};
-
-    use crate::chat::{self, create_group_chat, ProtectionStatus};
-    use crate::message::Message;
-    use crate::test_utils::{self, TestContext};
-
     use super::*;
 
+    use crate::chat::{create_group_chat, ProtectionStatus};
+    use crate::{
+        chat,
+        message::Message,
+        test_utils::{self, TestContext},
+    };
+    use anyhow::Result;
+    use image::Pixel;
+
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
@@ -940,7 +927,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_recode_image_1() {
+    async fn test_recode_image() {
         let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
         send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
@@ -957,10 +944,7 @@ mod tests {
         )
         .await
         .unwrap();
-    }
 
-    #[async_std::test]
-    async fn test_recode_image_2() {
         // The "-rotated" files are rotated by 270 degrees using the Exif metadata
         let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(
@@ -976,29 +960,22 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        let mut buf = Cursor::new(vec![]);
+        let mut bytes = vec![];
         img_rotated
-            .write_to(&mut buf, image::ImageFormat::Jpeg)
+            .write_to(&mut bytes, image::ImageFormat::Jpeg)
             .unwrap();
-        let bytes = buf.into_inner();
-
-        // Do this in parallel to speed up the test a bit
-        // (it still takes very long though)
-        let bytes2 = bytes.clone();
-        let join_handle = async_std::task::spawn(async move {
-            let img_rotated = send_image_check_mediaquality(
-                Some("0"),
-                &bytes2,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-                0,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-            )
-            .await
-            .unwrap();
-            assert_correct_rotation(&img_rotated);
-        });
+        let img_rotated = send_image_check_mediaquality(
+            Some("0"),
+            &bytes,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+            0,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+        assert_correct_rotation(&img_rotated);
 
         let img_rotated = send_image_check_mediaquality(
             Some("1"),
@@ -1013,11 +990,6 @@ mod tests {
         .unwrap();
         assert_correct_rotation(&img_rotated);
 
-        join_handle.await;
-    }
-
-    #[async_std::test]
-    async fn test_recode_image_3() {
         let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
         let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
             .await

src/chatlist.rs

@@ -1,14 +1,16 @@
 //! # Chat list module.
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{bail, ensure, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_GCL_ADD_ALLDONE_HINT,
-    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
+    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
+    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
-use crate::contact::{Contact, ContactId};
+use crate::contact::Contact;
 use crate::context::Context;
+use crate::ephemeral::delete_expired_messages;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
 use crate::summary::Summary;
@@ -83,13 +85,19 @@ impl Chatlist {
         context: &Context,
         listflags: usize,
         query: Option<&str>,
-        query_contact_id: Option<ContactId>,
+        query_contact_id: Option<u32>,
     ) -> Result<Self> {
         let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
         let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
         let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
         let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
 
+        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
+        // messages get deleted to avoid reloading the same chatlist.
+        if let Err(err) = delete_expired_messages(context).await {
+            warn!(context, "Failed to hide expired messages: {}", err);
+        }
+
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -104,7 +112,7 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, ContactId::DEVICE)
+            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
                 .await?
                 .unwrap_or_default()
         } else {
@@ -139,7 +147,7 @@ impl Chatlist {
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsv![MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned],
+                paramsv![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             ).await?
@@ -208,7 +216,7 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, ContactId::SELF)
+                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
                     .await?
                     .unwrap_or_default()
             } else {
@@ -263,23 +271,21 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> Result<ChatId> {
-        let (chat_id, _msg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
-        Ok(*chat_id)
+    pub fn get_chat_id(&self, index: usize) -> ChatId {
+        match self.ids.get(index) {
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
+        }
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
-        let (_chat_id, msg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
-        Ok(*msg_id)
+        match self.ids.get(index) {
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
+        }
     }
 
     /// Returns a summary for a given chatlist index.
@@ -293,10 +299,11 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let (chat_id, lastmsg_id) = self
-            .ids
-            .get(index)
-            .context("chatlist index is out of range")?;
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
+            Some(ids) => ids,
+            None => bail!("Chatlist index out of range"),
+        };
+
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
@@ -318,7 +325,7 @@ impl Chatlist {
 
         let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
             let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == ContactId::SELF {
+            if lastmsg.from_id == DC_CONTACT_ID_SELF {
                 (Some(lastmsg), None)
             } else {
                 match chat.typ {
@@ -335,7 +342,7 @@ impl Chatlist {
 
         if chat.id.is_archived_link() {
             Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != ContactId::UNDEFINED) {
+        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
             Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
         } else {
             Ok(Summary {
@@ -348,10 +355,6 @@ impl Chatlist {
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
         self.ids.iter().position(|(chat_id, _)| chat_id == &id)
     }
-
-    pub fn iter(&self) -> impl Iterator<Item = &(ChatId, Option<MsgId>)> {
-        self.ids.iter()
-    }
 }
 
 /// Returns the number of archived chats
@@ -371,8 +374,8 @@ mod tests {
     use super::*;
 
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
+    use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message::Viewtype;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
@@ -392,9 +395,9 @@ mod tests {
         // check that the chatlist starts with the most recent message
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id3);
-        assert_eq!(chats.get_chat_id(1).unwrap(), chat_id2);
-        assert_eq!(chats.get_chat_id(2).unwrap(), chat_id1);
+        assert_eq!(chats.get_chat_id(0), chat_id3);
+        assert_eq!(chats.get_chat_id(1), chat_id2);
+        assert_eq!(chats.get_chat_id(2), chat_id1);
 
         // New drafts are sorted to the top
         // We have to set a draft on the other two messages, too, as
@@ -411,7 +414,7 @@ mod tests {
         }
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id2);
+        assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
         let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
@@ -442,7 +445,7 @@ mod tests {
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -451,7 +454,7 @@ mod tests {
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -503,6 +506,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
                  \n\
                  hello foo\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -523,7 +527,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0).unwrap(), chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -563,6 +567,7 @@ mod tests {
                  Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
                  \n\
                  hello foo\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -578,7 +583,7 @@ mod tests {
         // check, the one-to-one-chat can be found using chatlist search query
         let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // change the name of the contact; this also changes the name of the one-to-one-chat
         let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
@@ -589,7 +594,7 @@ mod tests {
         assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
         let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, chat_id);
+        assert_eq!(chats.get_chat_id(0), chat_id);
 
         // revert name change, this again changes the name of the one-to-one-chat to the email-address
         let test_id = Contact::create(&t, "", "bob@example.org").await?;

src/config.rs

@@ -1,17 +1,20 @@
 //! # Key-value configuration management.
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{ensure, Result};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
+use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
-use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
+use crate::job;
+use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -64,18 +67,14 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MdnsEnabled,
 
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "1"))]
     SentboxWatch,
 
     #[strum(props(default = "1"))]
     MvboxMove,
 
-    /// Watch for new messages in the "Mvbox" (aka DeltaChat folder) only.
-    ///
-    /// This will not entirely disable other folders, e.g. the spam folder will also still
-    /// be watched for new messages.
     #[strum(props(default = "0"))]
-    OnlyFetchMvbox,
+    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
 
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
@@ -112,7 +111,6 @@ pub enum Config {
     DeleteDeviceAfter,
 
     SaveMimeHeaders,
-    /// The primary email address. Also see `SecondaryAddrs`.
     ConfiguredAddr,
     ConfiguredMailServer,
     ConfiguredMailUser,
@@ -131,14 +129,11 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
     ConfiguredTimestamp,
     ConfiguredProvider,
     Configured,
 
-    /// All secondary self addresses separated by spaces
-    /// (`addr1@example.org addr2@exapmle.org addr3@example.org`)
-    SecondaryAddrs,
-
     #[strum(serialize = "sys.version")]
     SysVersion,
 
@@ -205,6 +200,7 @@ impl Context {
 
         // Default values
         match key {
+            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
             Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
             _ => Ok(key.get_str("default").map(|s| s.to_string())),
         }
@@ -232,11 +228,6 @@ impl Context {
         Ok(self.get_config_int(key).await? != 0)
     }
 
-    pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
-        Ok(self.get_config_bool(Config::MvboxMove).await?
-            || self.get_config_bool(Config::OnlyFetchMvbox).await?)
-    }
-
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
@@ -245,7 +236,7 @@ impl Context {
         match self.get_config_int(Config::DeleteServerAfter).await? {
             0 => Ok(None),
             1 => Ok(Some(0)),
-            x => Ok(Some(i64::from(x))),
+            x => Ok(Some(x as i64)),
         }
     }
 
@@ -267,7 +258,7 @@ impl Context {
     pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
         match self.get_config_int(Config::DeleteDeviceAfter).await? {
             0 => Ok(None),
-            x => Ok(Some(i64::from(x))),
+            x => Ok(Some(x as i64)),
         }
     }
 
@@ -293,26 +284,55 @@ impl Context {
                     }
                 }
                 self.emit_event(EventType::SelfavatarChanged);
+                Ok(())
+            }
+            Config::Selfstatus => {
+                let def = stock_str::status_line(self).await;
+                let val = if value.is_none() || value.unwrap() == def {
+                    None
+                } else {
+                    value
+                };
+
+                self.sql.set_raw_config(key, val).await?;
+                Ok(())
             }
             Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(key, value).await;
-                // Interrupt ephemeral loop to delete old messages immediately.
-                self.interrupt_ephemeral_task().await;
-                ret?
+                let ret = self
+                    .sql
+                    .set_raw_config(key, value)
+                    .await
+                    .map_err(Into::into);
+                // Force chatlist reload to delete old messages immediately.
+                self.emit_event(EventType::MsgsChanged {
+                    msg_id: MsgId::new(0),
+                    chat_id: ChatId::new(0),
+                });
+                ret
             }
             Config::Displayname => {
                 let value = value.map(improve_single_line_input);
                 self.sql.set_raw_config(key, value.as_deref()).await?;
+                Ok(())
+            }
+            Config::DeleteServerAfter => {
+                let ret = self
+                    .sql
+                    .set_raw_config(key, value)
+                    .await
+                    .map_err(Into::into);
+                job::schedule_resync(self).await?;
+                ret
             }
             _ => {
                 self.sql.set_raw_config(key, value).await?;
+                Ok(())
             }
         }
-        Ok(())
     }
 
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { Some("0") })
+        self.set_config(key, if value { Some("1") } else { None })
             .await?;
         Ok(())
     }
@@ -333,73 +353,6 @@ impl Context {
     }
 }
 
-// Separate impl block for self address handling
-impl Context {
-    /// Determine whether the specified addr maps to the/a self addr.
-    /// Returns `false` if no addresses are configured.
-    pub(crate) async fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        Ok(self
-            .get_config(Config::ConfiguredAddr)
-            .await?
-            .iter()
-            .any(|a| addr_cmp(addr, a))
-            || self
-                .get_secondary_self_addrs()
-                .await?
-                .iter()
-                .any(|a| addr_cmp(addr, a)))
-    }
-
-    /// Sets `primary_new` as the new primary self address and saves the old
-    /// primary address (if exists) as a secondary address.
-    ///
-    /// This should only be used by test code and during configure.
-    pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        // add old primary address (if exists) to secondary addresses
-        let mut secondary_addrs = self.get_all_self_addrs().await?;
-        // never store a primary address also as a secondary
-        secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
-        self.set_config(
-            Config::SecondaryAddrs,
-            Some(secondary_addrs.join(" ").as_str()),
-        )
-        .await?;
-
-        self.set_config(Config::ConfiguredAddr, Some(primary_new))
-            .await?;
-
-        Ok(())
-    }
-
-    /// Returns all primary and secondary self addresses.
-    pub(crate) async fn get_all_self_addrs(&self) -> Result<Vec<String>> {
-        let primary_addrs = self.get_config(Config::ConfiguredAddr).await?.into_iter();
-        let secondary_addrs = self.get_secondary_self_addrs().await?.into_iter();
-
-        Ok(primary_addrs.chain(secondary_addrs).collect())
-    }
-
-    /// Returns all secondary self addresses.
-    pub(crate) async fn get_secondary_self_addrs(&self) -> Result<Vec<String>> {
-        let secondary_addrs = self
-            .get_config(Config::SecondaryAddrs)
-            .await?
-            .unwrap_or_default();
-        Ok(secondary_addrs
-            .split_ascii_whitespace()
-            .map(|s| s.to_string())
-            .collect())
-    }
-
-    /// Returns the primary self address.
-    /// Returns an error if no self addr is configured.
-    pub async fn get_primary_self_addr(&self) -> Result<String> {
-        self.get_config(Config::ConfiguredAddr)
-            .await?
-            .context("No self addr configured")
-    }
-}
-
 /// Returns all available configuration keys concated together.
 fn get_config_keys_string() -> String {
     let keys = Config::iter().fold(String::new(), |mut acc, key| {
@@ -471,70 +424,4 @@ mod tests {
 
         Ok(())
     }
-
-    /// Regression test for https://github.com/deltachat/deltachat-core-rust/issues/3012
-    #[async_std::test]
-    async fn test_set_config_bool() -> Result<()> {
-        let t = TestContext::new().await;
-
-        // We need some config that defaults to true
-        let c = Config::E2eeEnabled;
-        assert_eq!(t.get_config_bool(c).await?, true);
-        t.set_config_bool(c, false).await?;
-        assert_eq!(t.get_config_bool(c).await?, false);
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_self_addrs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["alice@example.org"]);
-        assert!(!alice.is_self_addr("alice@alice.com").await?);
-
-        // Test adding the same primary address
-        alice.set_primary_self_addr("alice@example.org").await?;
-        alice.set_primary_self_addr("Alice@Example.Org").await?;
-        assert_eq!(alice.get_all_self_addrs().await?, vec!["Alice@Example.Org"]);
-
-        // Test adding a new (primary) self address
-        // The address is trimmed during by `LoginParam::from_database()`,
-        // so `set_primary_self_addr()` doesn't have to trim it.
-        alice.set_primary_self_addr(" Alice@alice.com ").await?;
-        assert!(alice.is_self_addr("    aliCe@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec![" Alice@alice.com ", "Alice@Example.Org"]
-        );
-
-        // Check that the entry is not duplicated
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        alice.set_primary_self_addr("alice@alice.com").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.com", "Alice@Example.Org"]
-        );
-
-        // Test switching back
-        alice.set_primary_self_addr("alice@example.org").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@example.org", "alice@alice.com"]
-        );
-
-        // Test setting a new primary self address, the previous self address
-        // should be kept as a secondary self address
-        alice.set_primary_self_addr("alice@alice.xyz").await?;
-        assert_eq!(
-            alice.get_all_self_addrs().await?,
-            vec!["alice@alice.xyz", "alice@example.org", "alice@alice.com"]
-        );
-        assert!(alice.is_self_addr("alice@example.org").await?);
-        assert!(alice.is_self_addr("alice@alice.com").await?);
-        assert!(alice.is_self_addr("Alice@alice.xyz").await?);
-
-        Ok(())
-    }
 }

src/configure.rs

@@ -11,20 +11,21 @@ use async_std::task;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::config::Config;
-use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2};
-use crate::context::Context;
-use crate::dc_tools::{time, EmailAddress};
+use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
-use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
-use crate::param::Params;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::stock_str;
 use crate::{chat, e2ee, provider};
+use crate::{config::Config, dc_tools::time};
+use crate::{
+    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
+    job,
+};
+use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -85,7 +86,7 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::load_candidate_params(self).await?;
+        let mut param = LoginParam::from_database(self, "").await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
@@ -442,7 +443,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.should_watch_mvbox().await?;
+    let create_mvbox = ctx.get_config_bool(Config::MvboxMove).await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -453,14 +454,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     drop(imap);
 
     progress!(ctx, 910);
-
-    if ctx.get_config(Config::ConfiguredAddr).await?.as_deref() != Some(&param.addr) {
-        // Switched account, all server UIDs we know are invalid
-        job::schedule_resync(ctx).await?;
-    }
-
+    // configuration success - write back the configured parameters with the
+    // "configured_" prefix; also write the "configured"-flag */
     // the trailing underscore is correct
-    param.save_as_configured_params(ctx).await?;
+    param.save_to_database(ctx, "configured_").await?;
+    ctx.sql.set_raw_config_bool("configured", true).await?;
     ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
@@ -478,8 +476,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 940);
     update_device_chats_handle.await?;
 
-    ctx.sql.set_raw_config_bool("configured", true).await?;
-
     Ok(())
 }
 
@@ -705,11 +701,11 @@ pub enum Error {
         error: quick_xml::Error,
     },
 
+    #[error("Failed to get URL: {0}")]
+    ReadUrl(#[from] self::read_url::Error),
+
     #[error("Number of redirection is exceeded")]
     Redirection,
-
-    #[error("{0:#}")]
-    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]

src/configure/read_url.rs

@@ -1,40 +1,20 @@
 use crate::context::Context;
 
-use anyhow::format_err;
-use anyhow::Context as _;
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("URL request error")]
+    GetError(surf::Error),
+}
 
-pub async fn read_url(context: &Context, url: &str) -> anyhow::Result<String> {
-    match read_url_inner(context, url).await {
-        Ok(s) => {
-            info!(context, "Successfully read url {}", url);
-            Ok(s)
-        }
-        Err(e) => {
-            info!(context, "Can't read URL {}: {:#}", url, e);
-            Err(format_err!("Can't read URL {}: {:#}", url, e))
+pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+    info!(context, "Requesting URL {}", url);
+
+    match surf::get(url).recv_string().await {
+        Ok(res) => Ok(res),
+        Err(err) => {
+            info!(context, "Can\'t read URL {}: {}", url, err);
+
+            Err(Error::GetError(err))
         }
     }
 }
-
-pub async fn read_url_inner(context: &Context, mut url: &str) -> anyhow::Result<String> {
-    let mut _temp; // For the borrow checker
-
-    // Follow up to 10 http-redirects
-    for _i in 0..10 {
-        let mut response = surf::get(url).send().await.map_err(|e| e.into_inner())?;
-        if response.status().is_redirection() {
-            _temp = response
-                .header("location")
-                .context("Redirection doesn't have a target location")?
-                .last()
-                .to_string();
-            info!(context, "Following redirect to {}", _temp);
-            url = &_temp;
-            continue;
-        }
-
-        return response.body_string().await.map_err(|e| e.into_inner());
-    }
-
-    Err(format_err!("Followed 10 redirections"))
-}

src/configure/server_params.rs

@@ -52,8 +52,10 @@ impl ServerParams {
     fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
             vec![
-                // Try "imap.ex.org"/"smtp.ex.org" and "mail.ex.org" first because if a server exists
-                // under this address, it's likely the correct one.
+                Self {
+                    hostname: param_domain.to_string(),
+                    ..self.clone()
+                },
                 Self {
                     hostname: match self.protocol {
                         Protocol::Imap => "imap.".to_string() + param_domain,
@@ -63,12 +65,6 @@ impl ServerParams {
                 },
                 Self {
                     hostname: "mail.".to_string() + param_domain,
-                    ..self.clone()
-                },
-                // Try "ex.org" last because if it's wrong and the server is configured to
-                // not answer at all, configuration may be stuck for several minutes.
-                Self {
-                    hostname: param_domain.to_string(),
                     ..self
                 },
             ]
@@ -300,48 +296,5 @@ mod tests {
                 strict_tls: Some(true)
             }],
         );
-
-        // Test that "example.net" is tried after "*.example.net".
-        let v = expand_param_vector(
-            vec![ServerParams {
-                protocol: Protocol::Imap,
-                hostname: "".to_string(),
-                port: 10480,
-                socket: Socket::Ssl,
-                username: "foobar".to_string(),
-                strict_tls: Some(true),
-            }],
-            "foobar@example.net",
-            "example.net",
-        );
-        assert_eq!(
-            v,
-            vec![
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "imap.example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                },
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "mail.example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                },
-                ServerParams {
-                    protocol: Protocol::Imap,
-                    hostname: "example.net".to_string(),
-                    port: 10480,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: Some(true)
-                }
-            ],
-        );
     }
 }

src/constants.rs

@@ -179,6 +179,16 @@ pub const DC_ELLIPSIS: &str = "[...]";
 /// `char`s), not Unicode Grapheme Clusters.
 pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
+pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
+pub const DC_CONTACT_ID_SELF: u32 = 1;
+pub const DC_CONTACT_ID_INFO: u32 = 2;
+pub const DC_CONTACT_ID_DEVICE: u32 = 5;
+pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
+
+// decorative address that is used for DC_CONTACT_ID_DEVICE
+// when an api that returns an email is called.
+pub const DC_CONTACT_ID_DEVICE_ADDR: &str = "device@localhost";
+
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;
@@ -220,6 +230,79 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 // can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
 pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
 
+#[derive(
+    Debug,
+    Display,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
+    Serialize,
+    Deserialize,
+)]
+#[repr(u32)]
+pub enum Viewtype {
+    Unknown = 0,
+
+    /// Text message.
+    /// The text of the message is set using dc_msg_set_text()
+    /// and retrieved with dc_msg_get_text().
+    Text = 10,
+
+    /// Image message.
+    /// If the image is an animated GIF, the type DC_MSG_GIF should be used.
+    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
+    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
+    Image = 20,
+
+    /// Animated GIF message.
+    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
+    Gif = 21,
+
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker = 23,
+
+    /// Message containing an Audio file.
+    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
+    Audio = 40,
+
+    /// A voice message that was directly recorded by the user.
+    /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
+    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
+    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
+    Voice = 41,
+
+    /// Video messages.
+    /// File, width, height and durarion
+    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
+    /// and retrieved via
+    /// dc_msg_get_file(), dc_msg_get_width(),
+    /// dc_msg_get_height(), dc_msg_get_duration().
+    Video = 50,
+
+    /// Message containing any file, eg. a PDF.
+    /// The file is set via dc_msg_set_file()
+    /// and retrieved via dc_msg_get_file().
+    File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
+}
+
+impl Default for Viewtype {
+    fn default() -> Self {
+        Viewtype::Unknown
+    }
+}
+
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -231,9 +314,32 @@ pub enum KeyType {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     use num_traits::FromPrimitive;
 
-    use super::*;
+    #[test]
+    fn test_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+    }
 
     #[test]
     fn test_chattype_values() {

src/contact.rs

@@ -1,20 +1,21 @@
 //! Contacts module
 
 use std::convert::{TryFrom, TryInto};
-use std::fmt;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_std::path::PathBuf;
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use regex::Regex;
-use serde::{Deserialize, Serialize};
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
-use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
+use crate::constants::{
+    Blocked, Chattype, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR, DC_CONTACT_ID_LAST_SPECIAL,
+    DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
+};
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
 use crate::events::EventType;
@@ -24,94 +25,8 @@ use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
-use crate::sql::{self, params_iter};
 use crate::{chat, stock_str};
 
-/// Contact ID, including reserved IDs.
-///
-/// Some contact IDs are reserved to identify special contacts.  This
-/// type can represent both the special as well as normal contacts.
-#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
-pub struct ContactId(u32);
-
-impl ContactId {
-    pub const UNDEFINED: ContactId = ContactId::new(0);
-    /// The owner of the account.
-    ///
-    /// The email-address is set by `dc_set_config` using "addr".
-    pub const SELF: ContactId = ContactId::new(1);
-    pub const INFO: ContactId = ContactId::new(2);
-    pub const DEVICE: ContactId = ContactId::new(5);
-    const LAST_SPECIAL: ContactId = ContactId::new(9);
-
-    /// Address to go with [`ContactId::DEVICE`].
-    ///
-    /// This is used by APIs which need to return an email address for this contact.
-    pub const DEVICE_ADDR: &'static str = "device@localhost";
-
-    /// Creates a new [`ContactId`].
-    pub const fn new(id: u32) -> ContactId {
-        ContactId(id)
-    }
-
-    /// Whether this is a special [`ContactId`].
-    ///
-    /// Some [`ContactId`]s are reserved for special contacts like [`ContactId::SELF`],
-    /// [`ContactId::INFO`] and [`ContactId::DEVICE`].  This function indicates whether this
-    /// [`ContactId`] is any of the reserved special [`ContactId`]s (`true`) or whether it
-    /// is the [`ContactId`] of a real contact (`false`).
-    pub fn is_special(&self) -> bool {
-        self.0 <= Self::LAST_SPECIAL.0
-    }
-
-    /// Numerical representation of the [`ContactId`].
-    ///
-    /// Each contact ID has a unique numerical representation which is used in the database
-    /// (via [`rusqlite::ToSql`]) and also for FFI purposes.  In Rust code you should never
-    /// need to use this directly.
-    pub const fn to_u32(&self) -> u32 {
-        self.0
-    }
-}
-
-impl fmt::Display for ContactId {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if *self == ContactId::UNDEFINED {
-            write!(f, "Contact#Undefined")
-        } else if *self == ContactId::SELF {
-            write!(f, "Contact#Self")
-        } else if *self == ContactId::INFO {
-            write!(f, "Contact#Info")
-        } else if *self == ContactId::DEVICE {
-            write!(f, "Contact#Device")
-        } else if self.is_special() {
-            write!(f, "Contact#Special{}", self.0)
-        } else {
-            write!(f, "Contact#{}", self.0)
-        }
-    }
-}
-
-/// Allow converting [`ContactId`] to an SQLite type.
-impl rusqlite::types::ToSql for ContactId {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Integer(i64::from(self.0));
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
-/// Allow converting an SQLite integer directly into [`ContactId`].
-impl rusqlite::types::FromSql for ContactId {
-    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| {
-            val.try_into()
-                .map(ContactId::new)
-                .map_err(|_| rusqlite::types::FromSqlError::OutOfRange(val))
-        })
-    }
-}
-
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -127,7 +42,13 @@ impl rusqlite::types::FromSql for ContactId {
 #[derive(Debug)]
 pub struct Contact {
     /// The contact ID.
-    pub id: ContactId,
+    ///
+    /// Special message IDs:
+    /// - DC_CONTACT_ID_SELF (1) - this is the owner of the account with the email-address set by
+    ///   `dc_set_config` using "addr".
+    ///
+    /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
+    pub id: u32,
 
     /// Contact name. It is recommended to use `Contact::get_name`,
     /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
@@ -262,7 +183,7 @@ impl Default for VerifiedStatus {
 }
 
 impl Contact {
-    pub async fn load_from_db(context: &Context, contact_id: ContactId) -> Result<Self> {
+    pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
         let mut contact = context
             .sql
             .query_row(
@@ -270,7 +191,7 @@ impl Contact {
                 c.authname, c.param, c.status
                FROM contacts c
               WHERE c.id=?;",
-                paramsv![contact_id],
+                paramsv![contact_id as i32],
                 |row| {
                     let name: String = row.get(0)?;
                     let addr: String = row.get(1)?;
@@ -295,7 +216,7 @@ impl Contact {
                 },
             )
             .await?;
-        if contact_id == ContactId::SELF {
+        if contact_id == DC_CONTACT_ID_SELF {
             contact.name = stock_str::self_msg(context).await;
             contact.addr = context
                 .get_config(Config::ConfiguredAddr)
@@ -305,9 +226,9 @@ impl Contact {
                 .get_config(Config::Selfstatus)
                 .await?
                 .unwrap_or_default();
-        } else if contact_id == ContactId::DEVICE {
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
             contact.name = stock_str::device_messages(context).await;
-            contact.addr = ContactId::DEVICE_ADDR.to_string();
+            contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
             contact.status = stock_str::device_messages_hint(context).await;
         }
         Ok(contact)
@@ -324,18 +245,18 @@ impl Contact {
     }
 
     /// Check if a contact is blocked.
-    pub async fn is_blocked_load(context: &Context, id: ContactId) -> Result<bool> {
+    pub async fn is_blocked_load(context: &Context, id: u32) -> Result<bool> {
         let blocked = Self::load_from_db(context, id).await?.blocked;
         Ok(blocked)
     }
 
     /// Block the given contact.
-    pub async fn block(context: &Context, id: ContactId) -> Result<()> {
+    pub async fn block(context: &Context, id: u32) -> Result<()> {
         set_block_contact(context, id, true).await
     }
 
     /// Unblock the given contact.
-    pub async fn unblock(context: &Context, id: ContactId) -> Result<()> {
+    pub async fn unblock(context: &Context, id: u32) -> Result<()> {
         set_block_contact(context, id, false).await
     }
 
@@ -348,7 +269,7 @@ impl Contact {
     /// a bunch of addresses.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
-    pub async fn create(context: &Context, name: &str, addr: &str) -> Result<ContactId> {
+    pub async fn create(context: &Context, name: &str, addr: &str) -> Result<u32> {
         let name = improve_single_line_input(name);
         ensure!(!addr.is_empty(), "Cannot create contact with empty address");
 
@@ -371,12 +292,12 @@ impl Contact {
     }
 
     /// Mark messages from a contact as noticed.
-    pub async fn mark_noticed(context: &Context, id: ContactId) -> Result<()> {
+    pub async fn mark_noticed(context: &Context, id: u32) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE msgs SET state=? WHERE from_id=? AND state=?;",
-                paramsv![MessageState::InNoticed, id, MessageState::InFresh],
+                paramsv![MessageState::InNoticed, id as i32, MessageState::InFresh],
             )
             .await?;
         Ok(())
@@ -392,24 +313,29 @@ impl Contact {
         context: &Context,
         addr: &str,
         min_origin: Origin,
-    ) -> Result<Option<ContactId>> {
+    ) -> Result<Option<u32>> {
         if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
         }
 
         let addr_normalized = addr_normalize(addr);
 
-        if context.is_self_addr(addr_normalized).await? {
-            return Ok(Some(ContactId::SELF));
+        if let Some(addr_self) = context.get_config(Config::ConfiguredAddr).await? {
+            if addr_cmp(addr_normalized, &addr_self) {
+                return Ok(Some(DC_CONTACT_ID_SELF));
+            }
         }
-
         let id = context
             .sql
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
             AND id>?2 AND origin>=?3 AND blocked=0;",
-                paramsv![addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32,],
+                paramsv![
+                    addr_normalized,
+                    DC_CONTACT_ID_LAST_SPECIAL as i32,
+                    min_origin as u32,
+                ],
             )
             .await?;
         Ok(id)
@@ -445,16 +371,20 @@ impl Contact {
         name: &str,
         addr: &str,
         mut origin: Origin,
-    ) -> Result<(ContactId, Modifier)> {
+    ) -> Result<(u32, Modifier)> {
         let mut sth_modified = Modifier::None;
 
         ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
         let addr = addr_normalize(addr).to_string();
+        let addr_self = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
 
-        if context.is_self_addr(&addr).await? {
-            return Ok((ContactId::SELF, sth_modified));
+        if addr_cmp(&addr, &addr_self) {
+            return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -570,7 +500,7 @@ impl Contact {
                         paramsv![Chattype::Single, isize::try_from(row_id)?]
                     ).await?;
                     if let Some(chat_id) = chat_id {
-                        let contact = Contact::get_by_id(context, ContactId::new(row_id)).await?;
+                        let contact = Contact::get_by_id(context, row_id as u32).await?;
                         let chat_name = contact.get_display_name();
                         match context
                             .sql
@@ -627,7 +557,7 @@ impl Contact {
             }
         }
 
-        Ok((ContactId::new(row_id), sth_modified))
+        Ok((row_id, sth_modified))
     }
 
     /// Add a number of contacts.
@@ -687,8 +617,12 @@ impl Contact {
         context: &Context,
         listflags: u32,
         query: Option<impl AsRef<str>>,
-    ) -> Result<Vec<ContactId>> {
-        let self_addrs = context.get_all_self_addrs().await?;
+    ) -> Result<Vec<u32>> {
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
+
         let mut add_self = false;
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
@@ -699,46 +633,40 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    format!(
-                        "SELECT c.id FROM contacts c \
+                    "SELECT c.id FROM contacts c \
                  LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
-                 WHERE c.addr NOT IN ({})
-                 AND c.id>? \
-                 AND c.origin>=? \
+                 WHERE c.addr!=?1 \
+                 AND c.id>?2 \
+                 AND c.origin>=?3 \
                  AND c.blocked=0 \
-                 AND (iif(c.name='',c.authname,c.name) LIKE ? OR c.addr LIKE ?) \
-                 AND (1=? OR LENGTH(ps.verified_key_fingerprint)!=0)  \
+                 AND (iif(c.name='',c.authname,c.name) LIKE ?4 OR c.addr LIKE ?5) \
+                 AND (1=?6 OR LENGTH(ps.verified_key_fingerprint)!=0)  \
                  ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
-                        sql::repeat_vars(self_addrs.len())
-                    ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
-                        ContactId::LAST_SPECIAL,
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL as i32,
                         Origin::IncomingReplyTo,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 },
-                    ])),
-                    |row| row.get::<_, ContactId>(0),
+                    ],
+                    |row| row.get::<_, i32>(0),
                     |ids| {
                         for id in ids {
-                            ret.push(id?);
+                            ret.push(id? as u32);
                         }
                         Ok(())
                     },
                 )
                 .await?;
 
-            if let Some(query) = query {
-                let self_addr = context
-                    .get_config(Config::ConfiguredAddr)
-                    .await?
-                    .unwrap_or_default();
-                let self_name = context
-                    .get_config(Config::Displayname)
-                    .await?
-                    .unwrap_or_default();
-                let self_name2 = stock_str::self_msg(context);
+            let self_name = context
+                .get_config(Config::Displayname)
+                .await?
+                .unwrap_or_default();
+            let self_name2 = stock_str::self_msg(context);
 
+            if let Some(query) = query {
                 if self_addr.contains(query.as_ref())
                     || self_name.contains(query.as_ref())
                     || self_name2.await.contains(query.as_ref())
@@ -754,23 +682,21 @@ impl Contact {
             context
                 .sql
                 .query_map(
-                    format!(
-                        "SELECT id FROM contacts
-                 WHERE addr NOT IN ({})
-                 AND id>?
-                 AND origin>=?
+                    "SELECT id FROM contacts
+                 WHERE addr!=?1
+                 AND id>?2
+                 AND origin>=?3
                  AND blocked=0
                  ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                        sql::repeat_vars(self_addrs.len())
-                    ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
-                        ContactId::LAST_SPECIAL,
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL as i32,
                         Origin::IncomingReplyTo
-                    ])),
-                    |row| row.get::<_, ContactId>(0),
+                    ],
+                    |row| row.get::<_, i32>(0),
                     |ids| {
                         for id in ids {
-                            ret.push(id?);
+                            ret.push(id? as u32);
                         }
                         Ok(())
                     },
@@ -779,7 +705,7 @@ impl Contact {
         }
 
         if flag_add_self && add_self {
-            ret.push(ContactId::SELF);
+            ret.push(DC_CONTACT_ID_SELF);
         }
 
         Ok(ret)
@@ -834,14 +760,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0",
-                paramsv![ContactId::LAST_SPECIAL],
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
             )
             .await?;
         Ok(count as usize)
     }
 
     /// Get blocked contacts.
-    pub async fn get_all_blocked(context: &Context) -> Result<Vec<ContactId>> {
+    pub async fn get_all_blocked(context: &Context) -> Result<Vec<u32>> {
         Contact::update_blocked_mailinglist_contacts(context)
             .await
             .context("cannot update blocked mailinglist contacts")?;
@@ -850,8 +776,8 @@ impl Contact {
             .sql
             .query_map(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                paramsv![ContactId::LAST_SPECIAL],
-                |row| row.get::<_, ContactId>(0),
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
+                |row| row.get::<_, u32>(0),
                 |ids| {
                     ids.collect::<std::result::Result<Vec<_>, _>>()
                         .map_err(Into::into)
@@ -866,15 +792,15 @@ impl Contact {
     /// This function returns a string explaining the encryption state
     /// of the contact and if the connection is encrypted the
     /// fingerprints of the keys involved.
-    pub async fn get_encrinfo(context: &Context, contact_id: ContactId) -> Result<String> {
+    pub async fn get_encrinfo(context: &Context, contact_id: u32) -> Result<String> {
         ensure!(
-            !contact_id.is_special(),
+            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
             "Can not provide encryption info for special contact"
         );
 
         let mut ret = String::new();
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let loginparam = LoginParam::load_configured_params(context).await?;
+            let loginparam = LoginParam::from_database(context, "configured_").await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
             if let Some(peerstate) = peerstate.filter(|peerstate| {
@@ -935,21 +861,27 @@ impl Contact {
     /// possible as the contact is in use. In this case, the contact can be blocked.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
-    pub async fn delete(context: &Context, contact_id: ContactId) -> Result<()> {
-        ensure!(!contact_id.is_special(), "Can not delete special contact");
+    pub async fn delete(context: &Context, contact_id: u32) -> Result<()> {
+        ensure!(
+            contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+            "Can not delete special contact"
+        );
 
         let count_chats = context
             .sql
             .count(
                 "SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?;",
-                paramsv![contact_id],
+                paramsv![contact_id as i32],
             )
             .await?;
 
         if count_chats == 0 {
             match context
                 .sql
-                .execute("DELETE FROM contacts WHERE id=?;", paramsv![contact_id])
+                .execute(
+                    "DELETE FROM contacts WHERE id=?;",
+                    paramsv![contact_id as i32],
+                )
                 .await
             {
                 Ok(_) => {
@@ -972,10 +904,10 @@ impl Contact {
 
     /// Get a single contact object.  For a list, see eg. dc_get_contacts().
     ///
-    /// For contact ContactId::SELF (1), the function returns sth.
+    /// For contact DC_CONTACT_ID_SELF (1), the function returns sth.
     /// like "Me" in the selected language and the email address
     /// defined by dc_set_config().
-    pub async fn get_by_id(context: &Context, contact_id: ContactId) -> Result<Contact> {
+    pub async fn get_by_id(context: &Context, contact_id: u32) -> Result<Contact> {
         let contact = Contact::load_from_db(context, contact_id).await?;
 
         Ok(contact)
@@ -987,7 +919,7 @@ impl Contact {
             .sql
             .execute(
                 "UPDATE contacts SET param=? WHERE id=?",
-                paramsv![self.param.to_string(), self.id],
+                paramsv![self.param.to_string(), self.id as i32],
             )
             .await?;
         Ok(())
@@ -999,14 +931,14 @@ impl Contact {
             .sql
             .execute(
                 "UPDATE contacts SET status=? WHERE id=?",
-                paramsv![self.status, self.id],
+                paramsv![self.status, self.id as i32],
             )
             .await?;
         Ok(())
     }
 
     /// Get the ID of the contact.
-    pub fn get_id(&self) -> ContactId {
+    pub fn get_id(&self) -> u32 {
         self.id
     }
 
@@ -1065,7 +997,7 @@ impl Contact {
     /// This is the image set by each remote user on their own
     /// using dc_set_config(context, "selfavatar", image).
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
-        if self.id == ContactId::SELF {
+        if self.id == DC_CONTACT_ID_SELF {
             if let Some(p) = context.get_config(Config::Selfavatar).await? {
                 return Ok(Some(PathBuf::from(p)));
             }
@@ -1111,7 +1043,7 @@ impl Contact {
     ) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
-        if self.id == ContactId::SELF {
+        if self.id == DC_CONTACT_ID_SELF {
             return Ok(VerifiedStatus::BidirectVerified);
         }
 
@@ -1130,6 +1062,26 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
+    pub async fn addr_equals_contact(
+        context: &Context,
+        addr: &str,
+        contact_id: u32,
+    ) -> Result<bool> {
+        if addr.is_empty() {
+            return Ok(false);
+        }
+
+        let contact = Contact::load_from_db(context, contact_id).await?;
+        if !contact.addr.is_empty() {
+            let normalized_addr = addr_normalize(addr);
+            if contact.addr == normalized_addr {
+                return Ok(true);
+            }
+        }
+
+        Ok(false)
+    }
+
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
             return Ok(0);
@@ -1139,14 +1091,14 @@ impl Contact {
             .sql
             .count(
                 "SELECT COUNT(*) FROM contacts WHERE id>?;",
-                paramsv![ContactId::LAST_SPECIAL],
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
             )
             .await?;
         Ok(count)
     }
 
-    pub async fn real_exists_by_id(context: &Context, contact_id: ContactId) -> Result<bool> {
-        if contact_id.is_special() {
+    pub async fn real_exists_by_id(context: &Context, contact_id: u32) -> Result<bool> {
+        if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return Ok(false);
         }
 
@@ -1154,7 +1106,7 @@ impl Contact {
             .sql
             .exists(
                 "SELECT COUNT(*) FROM contacts WHERE id=?;",
-                paramsv![contact_id],
+                paramsv![contact_id as i32],
             )
             .await?;
         Ok(exists)
@@ -1162,14 +1114,14 @@ impl Contact {
 
     pub async fn scaleup_origin_by_id(
         context: &Context,
-        contact_id: ContactId,
+        contact_id: u32,
         origin: Origin,
     ) -> Result<()> {
         context
             .sql
             .execute(
                 "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
-                paramsv![origin, contact_id, origin],
+                paramsv![origin, contact_id as i32, origin],
             )
             .await?;
         Ok(())
@@ -1213,13 +1165,9 @@ fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     }
 }
 
-async fn set_block_contact(
-    context: &Context,
-    contact_id: ContactId,
-    new_blocking: bool,
-) -> Result<()> {
+async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) -> Result<()> {
     ensure!(
-        !contact_id.is_special(),
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
         "Can't block special contact {}",
         contact_id
     );
@@ -1231,7 +1179,7 @@ async fn set_block_contact(
             .sql
             .execute(
                 "UPDATE contacts SET blocked=? WHERE id=?;",
-                paramsv![i32::from(new_blocking), contact_id],
+                paramsv![new_blocking as i32, contact_id as i32],
             )
             .await?;
 
@@ -1282,14 +1230,14 @@ WHERE type=? AND id IN (
 /// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     profile_image: &AvatarAction,
     was_encrypted: bool,
 ) -> Result<()> {
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            if contact_id == ContactId::SELF {
+            if contact_id == DC_CONTACT_ID_SELF {
                 if was_encrypted {
                     context
                         .set_config(Config::Selfavatar, Some(profile_image))
@@ -1303,7 +1251,7 @@ pub(crate) async fn set_profile_image(
             true
         }
         AvatarAction::Delete => {
-            if contact_id == ContactId::SELF {
+            if contact_id == DC_CONTACT_ID_SELF {
                 if was_encrypted {
                     context.set_config(Config::Selfavatar, None).await?;
                 } else {
@@ -1329,12 +1277,12 @@ pub(crate) async fn set_profile_image(
 /// between Delta Chat devices.
 pub(crate) async fn set_status(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     status: String,
     encrypted: bool,
     has_chat_version: bool,
 ) -> Result<()> {
-    if contact_id == ContactId::SELF {
+    if contact_id == DC_CONTACT_ID_SELF {
         if encrypted && has_chat_version {
             context
                 .set_config(Config::Selfstatus, Some(&status))
@@ -1355,11 +1303,11 @@ pub(crate) async fn set_status(
 /// Updates last seen timestamp of the contact if it is earlier than the given `timestamp`.
 pub(crate) async fn update_last_seen(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     timestamp: i64,
 ) -> Result<()> {
     ensure!(
-        !contact_id.is_special(),
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
         "Can not update special contact last seen timestamp"
     );
 
@@ -1420,6 +1368,17 @@ fn cat_fingerprint(
     }
 }
 
+impl Context {
+    /// determine whether the specified addr maps to the/a self addr
+    pub async fn is_self_addr(&self, addr: &str) -> Result<bool> {
+        if let Some(self_addr) = self.get_config(Config::ConfiguredAddr).await? {
+            Ok(addr_cmp(&self_addr, addr))
+        } else {
+            Ok(false)
+        }
+    }
+}
+
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
     let norm1 = addr_normalize(addr1).to_lowercase();
     let norm2 = addr_normalize(addr2).to_lowercase();
@@ -1452,17 +1411,6 @@ mod tests {
     use crate::message::Message;
     use crate::test_utils::{self, TestContext};
 
-    #[test]
-    fn test_contact_id_values() {
-        // Some FFI users need to have the values of these fixed, how naughty.  But let's
-        // make sure we don't modify them anyway.
-        assert_eq!(ContactId::UNDEFINED.to_u32(), 0);
-        assert_eq!(ContactId::SELF.to_u32(), 1);
-        assert_eq!(ContactId::INFO.to_u32(), 2);
-        assert_eq!(ContactId::DEVICE.to_u32(), 5);
-        assert_eq!(ContactId::LAST_SPECIAL.to_u32(), 9);
-    }
-
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1515,8 +1463,6 @@ mod tests {
     async fn test_get_contacts() -> Result<()> {
         let context = TestContext::new().await;
 
-        assert!(context.get_all_self_addrs().await?.is_empty());
-
         // Bob is not in the contacts yet.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1528,7 +1474,7 @@ mod tests {
             Origin::IncomingReplyTo,
         )
         .await?;
-        assert_ne!(id, ContactId::UNDEFINED);
+        assert_ne!(id, 0);
 
         let contact = Contact::load_from_db(&context.ctx, id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1606,7 +1552,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_id(), contact_id);
@@ -1633,7 +1579,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "three@drei.sam", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "");
@@ -1673,7 +1619,7 @@ mod tests {
             Contact::add_or_lookup(&t, "", "alice@w.de", Origin::IncomingUnknownTo)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::None);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_name(), "Wonderland, Alice");
@@ -1682,7 +1628,8 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
 
         // check SELF
-        let contact = Contact::load_from_db(&t, ContactId::SELF).await.unwrap();
+        let contact = Contact::load_from_db(&t, DC_CONTACT_ID_SELF).await.unwrap();
+        assert_eq!(DC_CONTACT_ID_SELF, 1);
         assert_eq!(contact.get_name(), stock_str::self_msg(&t).await);
         assert_eq!(contact.get_addr(), ""); // we're not configured
         assert!(!contact.is_blocked());
@@ -1692,7 +1639,7 @@ mod tests {
     async fn test_delete() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
-        assert!(Contact::delete(&alice, ContactId::SELF).await.is_err());
+        assert!(Contact::delete(&alice, DC_CONTACT_ID_SELF).await.is_err());
 
         // Create Bob contact
         let (contact_id, _) =
@@ -1725,7 +1672,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob1", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Created);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob1");
@@ -1737,7 +1684,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob2", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
@@ -1748,7 +1695,7 @@ mod tests {
         let contact_id = Contact::create(&t, "bob3", "bob@example.org")
             .await
             .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
@@ -1759,7 +1706,7 @@ mod tests {
             Contact::add_or_lookup(&t, "bob4", "bob@example.org", Origin::IncomingUnknownFrom)
                 .await
                 .unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "bob4");
@@ -1773,7 +1720,7 @@ mod tests {
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t, "", "claire@example.org").await.unwrap();
-        assert!(!contact_id.is_special());
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
         let contact = Contact::load_from_db(&t, contact_id).await.unwrap();
         assert_eq!(contact.get_authname(), "");
         assert_eq!(contact.get_name(), "");
@@ -1947,7 +1894,7 @@ mod tests {
         let id = Contact::lookup_id_by_addr(&alice.ctx, "alice@example.org", Origin::Unknown)
             .await
             .unwrap();
-        assert_eq!(id, Some(ContactId::SELF));
+        assert_eq!(id, Some(DC_CONTACT_ID_SELF));
     }
 
     #[async_std::test]
@@ -1955,9 +1902,9 @@ mod tests {
         let alice = TestContext::new_alice().await;
 
         // Return error for special IDs
-        let encrinfo = Contact::get_encrinfo(&alice, ContactId::SELF).await;
+        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_SELF).await;
         assert!(encrinfo.is_err());
-        let encrinfo = Contact::get_encrinfo(&alice, ContactId::DEVICE).await;
+        let encrinfo = Contact::get_encrinfo(&alice, DC_CONTACT_ID_DEVICE).await;
         assert!(encrinfo.is_err());
 
         let (contact_bob_id, _modified) =
@@ -2138,7 +2085,7 @@ Chat-Version: 1.0
 Date: Sun, 22 Mar 2020 22:37:55 +0000
 
 Hi."#;
-        dc_receive_imf(&alice, mime, false).await?;
+        dc_receive_imf(&alice, mime, "Inbox", false).await?;
         let msg = alice.get_last_msg().await;
 
         let timestamp = msg.get_timestamp();

src/context.rs

@@ -10,6 +10,7 @@ use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
+    task,
 };
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -23,6 +24,7 @@ use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
 use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
+use crate::securejoin::Bob;
 use crate::sql::Sql;
 
 #[derive(Clone, Debug)]
@@ -40,9 +42,12 @@ impl Deref for Context {
 
 #[derive(Debug)]
 pub struct InnerContext {
+    /// Database file path
+    pub(crate) dbfile: PathBuf,
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
+    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -55,6 +60,7 @@ pub struct InnerContext {
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
     /// Recently loaded quota information, if any.
     /// Set to `None` if quota was never tried to load.
@@ -78,7 +84,7 @@ pub struct InnerContext {
 
 #[derive(Debug)]
 pub struct RunningState {
-    ongoing_running: bool,
+    pub ongoing_running: bool,
     shall_stop_ongoing: bool,
     cancel_sender: Option<Sender<()>>,
 }
@@ -100,19 +106,10 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 }
 
 impl Context {
-    /// Creates new context and opens the database.
+    /// Creates new context.
     pub async fn new(dbfile: PathBuf, id: u32) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id).await?;
+        // pretty_env_logger::try_init_timed().ok();
 
-        // Open the database if is not encrypted.
-        if context.check_passphrase("".to_string()).await? {
-            context.sql.open(&context, "".to_string()).await?;
-        }
-        Ok(context)
-    }
-
-    /// Creates new context without opening the database.
-    pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -120,35 +117,7 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile, blobdir, id).await?;
-        Ok(context)
-    }
-
-    /// Opens the database with the given passphrase.
-    ///
-    /// Returns true if passphrase is correct, false is passphrase is not correct. Fails on other
-    /// errors.
-    pub async fn open(&self, passphrase: String) -> Result<bool> {
-        if self.sql.check_passphrase(passphrase.clone()).await? {
-            self.sql.open(self, passphrase).await?;
-            Ok(true)
-        } else {
-            Ok(false)
-        }
-    }
-
-    /// Returns true if database is open.
-    pub async fn is_open(&self) -> bool {
-        self.sql.is_open().await
-    }
-
-    /// Tests the database passphrase.
-    ///
-    /// Returns true if passphrase is correct.
-    ///
-    /// Fails if database is already open.
-    pub(crate) async fn check_passphrase(&self, passphrase: String) -> Result<bool> {
-        self.sql.check_passphrase(passphrase).await
+        Context::with_blobdir(dbfile, blobdir, id).await
     }
 
     pub(crate) async fn with_blobdir(
@@ -165,8 +134,10 @@ impl Context {
         let inner = InnerContext {
             id,
             blobdir,
+            dbfile,
             running_state: RwLock::new(Default::default()),
-            sql: Sql::new(dbfile),
+            sql: Sql::new(),
+            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -174,6 +145,7 @@ impl Context {
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -183,35 +155,32 @@ impl Context {
         let ctx = Context {
             inner: Arc::new(inner),
         };
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
 
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
-        if let Ok(false) = self.is_configured().await {
-            warn!(self, "can not start io on a context that is not configured");
+        info!(self, "starting IO");
+        if self.inner.is_io_running().await {
+            info!(self, "IO is already running");
             return;
         }
 
-        info!(self, "starting IO");
-        if let Err(err) = self.inner.scheduler.write().await.start(self.clone()).await {
-            error!(self, "Failed to start IO: {}", err)
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            if let Err(err) = l.start(self.clone()).await {
+                error!(self, "Failed to start IO: {}", err)
+            }
         }
     }
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
-        // Sending an event wakes up event pollers (get_next_event)
-        // so the caller of stop_io() can arrange for proper termination.
-        // For this, the caller needs to instruct the event poller
-        // to terminate on receiving the next event and then call stop_io()
-        // which will emit the below event(s)
         info!(self, "stopping IO");
 
-        if let Err(err) = self.inner.stop_io().await {
-            warn!(self, "failed to stop IO: {}", err);
-        }
+        self.inner.stop_io().await;
     }
 
     /// Returns a reference to the underlying SQL instance.
@@ -224,7 +193,7 @@ impl Context {
 
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
-        self.sql.dbfile.as_path()
+        self.dbfile.as_path()
     }
 
     /// Returns blob directory path.
@@ -240,24 +209,6 @@ impl Context {
         });
     }
 
-    /// Emits a generic MsgsChanged event (without chat or message id)
-    pub fn emit_msgs_changed_without_ids(&self) {
-        self.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
-    }
-
-    /// Emits a MsgsChanged event with specified chat and message ids
-    pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
-        self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
-    }
-
-    /// Emits an IncomingMsg event with specified chat and message ids
-    pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
-        self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
-    }
-
     /// Returns a receiver for emitted events.
     ///
     /// Multiple emitters can be created, but note that in this case each emitted event will
@@ -273,7 +224,7 @@ impl Context {
 
     // Ongoing process allocation/free/check
 
-    pub(crate) async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
+    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
         if self.has_ongoing().await {
             bail!("There is already another ongoing process running.");
         }
@@ -289,7 +240,7 @@ impl Context {
         Ok(receiver)
     }
 
-    pub(crate) async fn free_ongoing(&self) {
+    pub async fn free_ongoing(&self) {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
 
@@ -298,7 +249,7 @@ impl Context {
         s.cancel_sender.take();
     }
 
-    pub(crate) async fn has_ongoing(&self) -> bool {
+    pub async fn has_ongoing(&self) -> bool {
         let s_a = &self.running_state;
         let s = s_a.read().await;
 
@@ -323,7 +274,7 @@ impl Context {
         };
     }
 
-    pub(crate) async fn shall_stop_ongoing(&self) -> bool {
+    pub async fn shall_stop_ongoing(&self) -> bool {
         self.running_state.read().await.shall_stop_ongoing
     }
 
@@ -333,9 +284,8 @@ impl Context {
 
     pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::load_candidate_params(self).await?;
-        let l2 = LoginParam::load_configured_params(self).await?;
-        let secondary_addrs = self.get_secondary_self_addrs().await?.join(", ");
+        let l = LoginParam::from_database(self, "").await?;
+        let l2 = LoginParam::from_database(self, "configured_").await?;
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
         let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
@@ -374,7 +324,7 @@ impl Context {
 
         let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
         let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
-        let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
+        let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
         let folders_configured = self
             .sql
             .get_raw_config_int("folders_configured")
@@ -400,13 +350,6 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert(
-            "database_encrypted",
-            self.sql
-                .is_encrypted()
-                .await
-                .map_or_else(|| "closed".to_string(), |b| b.to_string()),
-        );
         res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
@@ -420,7 +363,6 @@ impl Context {
         res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
-        res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
             self.get_config_int(Config::FetchExistingMsgs)
@@ -439,7 +381,7 @@ impl Context {
         );
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
-        res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
+        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
@@ -627,14 +569,19 @@ impl Context {
         Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub(crate) fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
+        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
+        Ok(spam.as_deref() == Some(folder_name))
+    }
+
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
 
-    pub(crate) fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
         let mut wal_fname = OsString::new();
         wal_fname.push(dbfile.file_name().unwrap_or_default());
         wal_fname.push("-wal");
@@ -643,9 +590,25 @@ impl Context {
 }
 
 impl InnerContext {
-    async fn stop_io(&self) -> Result<()> {
-        self.scheduler.write().await.stop().await?;
-        Ok(())
+    async fn is_io_running(&self) -> bool {
+        self.scheduler.read().await.is_running()
+    }
+
+    async fn stop_io(&self) {
+        if self.is_io_running().await {
+            let token = {
+                let lock = &*self.scheduler.read().await;
+                lock.pre_stop().await
+            };
+            {
+                let lock = &mut *self.scheduler.write().await;
+                lock.stop(token).await;
+            }
+        }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
+        }
     }
 }
 
@@ -670,26 +633,21 @@ mod tests {
     use crate::chat::{
         get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
-    use crate::contact::ContactId;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
-    use crate::message::{Message, Viewtype};
+    use crate::message::Message;
     use crate::test_utils::TestContext;
-    use anyhow::Context as _;
     use std::time::Duration;
     use strum::IntoEnumIterator;
-    use tempfile::tempdir;
 
     #[async_std::test]
-    async fn test_wrong_db() -> Result<()> {
-        let tmp = tempfile::tempdir()?;
+    async fn test_wrong_db() {
+        let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        std::fs::write(&dbfile, b"123")?;
-        let res = Context::new(dbfile.into(), 1).await?;
-
-        // Broken database is indistinguishable from encrypted one.
-        assert_eq!(res.is_open().await, false);
-        Ok(())
+        std::fs::write(&dbfile, b"123").unwrap();
+        let res = Context::new(dbfile.into(), 1).await;
+        assert!(res.is_err());
     }
 
     #[async_std::test]
@@ -716,7 +674,9 @@ mod tests {
             dc_create_outgoing_rfc724_mid(None, contact.get_addr())
         );
         println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), false).await.unwrap();
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", false)
+            .await
+            .unwrap();
     }
 
     #[async_std::test]
@@ -954,7 +914,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;
@@ -1039,29 +999,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_check_passphrase() -> Result<()> {
-        let dir = tempdir()?;
-        let dbfile = dir.path().join("db.sqlite");
-
-        let id = 1;
-        let context = Context::new_closed(dbfile.clone().into(), id)
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.open("foo".to_string()).await?, true);
-        assert_eq!(context.is_open().await, true);
-        drop(context);
-
-        let id = 2;
-        let context = Context::new(dbfile.into(), id)
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.is_open().await, false);
-        assert_eq!(context.check_passphrase("bar".to_string()).await?, false);
-        assert_eq!(context.open("false".to_string()).await?, false);
-        assert_eq!(context.open("foo".to_string()).await?, true);
-
-        Ok(())
-    }
 }

src/dc_tools.rs

@@ -13,7 +13,7 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::Error;
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use mailparse::dateparse;
 use mailparse::headers::Headers;
@@ -21,10 +21,10 @@ use mailparse::MailHeaderMap;
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::provider::get_provider_update_timestamp;
 use crate::stock_str;
 
@@ -71,7 +71,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
     let lt = Local::now();
-    i64::from(lt.offset().local_minus_utc())
+    lt.offset().local_minus_utc() as i64
 }
 
 // timesmearing
@@ -86,7 +86,7 @@ pub(crate) fn dc_gm2local_offset() -> i64 {
 //   `last_smeared_timestamp` is again in sync with the normal time.
 // - however, we do not do all this for the far future,
 //   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
-pub(crate) const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 
 /// Returns the current smeared timestamp,
 ///
@@ -195,29 +195,44 @@ async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time
 }
 
 /* Message-ID tools */
-
-/// Generate an ID. The generated ID should be as short and as unique as possible:
-/// - short, because it may also used as part of Message-ID headers or in QR codes
-/// - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
-/// IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
-///
-/// Additional information when used as a message-id or group-id:
-/// - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
-/// - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
-/// - the group-id should be a string with the characters [a-zA-Z0-9\-_]
 pub(crate) fn dc_create_id() -> String {
-    // ThreadRng implements CryptoRng trait and is supposed to be cryptographically secure.
+    /* generate an id. the generated ID should be as short and as unique as possible:
+    - short, because it may also used as part of Message-ID headers or in QR codes
+    - unique as two IDs generated on two devices should not be the same. However, collisions are not world-wide but only by the few contacts.
+    IDs generated by this function are 66 bit wide and are returned as 11 base64 characters.
+    If possible, RNG of OpenSSL is used.
+
+    Additional information when used as a message-id or group-id:
+    - for OUTGOING messages this ID is written to the header as `Chat-Group-ID:` and is added to the message ID as Gr.<grpid>.<random>@<random>
+    - for INCOMING messages, the ID is taken from the Chat-Group-ID-header or from the Message-ID in the In-Reply-To: or References:-Header
+    - the group-id should be a string with the characters [a-zA-Z0-9\-_] */
+
     let mut rng = thread_rng();
+    let buf: [u32; 3] = [rng.gen(), rng.gen(), rng.gen()];
 
-    // Generate 72 random bits.
-    let mut arr = [0u8; 9];
-    rng.fill(&mut arr[..]);
+    encode_66bits_as_base64(buf[0usize], buf[1usize], buf[2usize])
+}
 
-    // Take 11 base64 characters containing 66 random bits.
-    base64::encode_config(&arr, base64::URL_SAFE)
-        .chars()
-        .take(11)
-        .collect()
+/// Encode 66 bits as a base64 string.
+/// This is useful for ID generating with short strings as we save 5 character
+/// in each id compared to 64 bit hex encoding. For a typical group ID, these
+/// are 10 characters (grpid+msgid):
+///    hex:    64 bit, 4 bits/character, length = 64/4 = 16 characters
+///    base64: 64 bit, 6 bits/character, length = 64/6 = 11 characters (plus 2 additional bits)
+/// Only the lower 2 bits of `fill` are used.
+fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
+    use byteorder::{BigEndian, WriteBytesExt};
+
+    let mut wrapped_writer = Vec::new();
+    {
+        let mut enc = base64::write::EncoderWriter::new(&mut wrapped_writer, base64::URL_SAFE);
+        enc.write_u32::<BigEndian>(v1).unwrap();
+        enc.write_u32::<BigEndian>(v2).unwrap();
+        enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
+        enc.finish().unwrap();
+    }
+    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
+    String::from_utf8(wrapped_writer).unwrap()
 }
 
 /// Function generates a Message-ID that can be used for a new outgoing message.
@@ -339,6 +354,63 @@ pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
     }
 }
 
+pub(crate) async fn dc_copy_file(
+    context: &Context,
+    src_path: impl AsRef<Path>,
+    dest_path: impl AsRef<Path>,
+) -> bool {
+    let src_abs = dc_get_abs_path(context, &src_path);
+    let mut src_file = match fs::File::open(&src_abs).await {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for read '{}': {}",
+                src_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    let dest_abs = dc_get_abs_path(context, &dest_path);
+    let mut dest_file = match fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&dest_abs)
+        .await
+    {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for write '{}': {}",
+                dest_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    match io::copy(&mut src_file, &mut dest_file).await {
+        Ok(_) => true,
+        Err(err) => {
+            error!(
+                context,
+                "Cannot copy \"{}\" to \"{}\": {}",
+                src_abs.display(),
+                dest_abs.display(),
+                err
+            );
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                fs::remove_file(dest_abs).await.ok();
+            }
+            false
+        }
+    }
+}
+
 pub(crate) async fn dc_create_folder(
     context: &Context,
     path: impl AsRef<Path>,
@@ -436,6 +508,33 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -537,8 +636,8 @@ impl rusqlite::types::ToSql for EmailAddress {
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
 pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .replace('\n', " ")
-        .replace('\r', " ")
+        .replace("\n", " ")
+        .replace("\r", " ")
         .trim()
         .to_string()
 }
@@ -703,7 +802,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
     async fn check_parse_receive_headers_integration(raw: &[u8], expected: &str) {
         let t = TestContext::new_alice().await;
         t.set_config(Config::ShowEmails, Some("2")).await.unwrap();
-        dc_receive_imf(&t, raw, false).await.unwrap();
+        dc_receive_imf(&t, raw, "INBOX", false).await.unwrap();
         let msg = t.get_last_msg().await;
         let msg_info = get_msg_info(&t, msg.id).await.unwrap();
 
@@ -766,12 +865,23 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
     }
 
     #[test]
-    fn test_dc_create_id_invalid_chars() {
-        for _ in 1..1000 {
-            let buf = dc_create_id();
-            assert!(!buf.contains('/')); // `/` must not be used to be URL-safe
-            assert!(!buf.contains('.')); // `.` is used as a delimiter when extracting grpid from Message-ID
-        }
+    fn test_encode_66bits_as_base64() {
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 0),
+            "ASNFZ4mrze8"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 1),
+            "ASNFZ4mrze9"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 2),
+            "ASNFZ4mrze-"
+        );
+        assert_eq!(
+            encode_66bits_as_base64(0x01234567, 0x89abcdef, 3),
+            "ASNFZ4mrze_"
+        );
     }
 
     #[test]
@@ -915,7 +1025,20 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
 
         assert!(dc_file_exist!(context, &abs_path).await);
 
+        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+
+        // attempting to copy a second time should fail
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
+
+        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
+
+        assert_eq!(buf.len(), 7);
+        assert_eq!(&buf, b"content");
+
         assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
+        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
             .await
             .is_ok());
@@ -1041,7 +1164,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1079,7 +1202,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        assert_eq!(device_chat_id, chats.get_chat_id(0).unwrap());
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1111,7 +1234,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1133,7 +1256,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();
@@ -1150,7 +1273,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22
         .await;
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0).unwrap();
+        let device_chat_id = chats.get_chat_id(0);
         let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
             .await
             .unwrap();

src/download.rs

@@ -3,14 +3,14 @@
 use anyhow::{anyhow, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
 
 use crate::config::Config;
+use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
-use crate::message::{Message, MsgId, Viewtype};
+use crate::message::{Message, MsgId};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::param::Params;
 use crate::{job_try, stock_str, EventType};
@@ -146,7 +146,7 @@ impl Job {
 
         if let Some((server_uid, server_folder)) = row {
             match imap
-                .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
+                .fetch_single_msg(context, &server_folder, server_uid)
                 .await
             {
                 ImapActionResult::RetryLater | ImapActionResult::Failed => {
@@ -185,7 +185,6 @@ impl Imap {
         context: &Context,
         folder: &str,
         uid: u32,
-        rfc724_mid: String,
     ) -> ImapActionResult {
         if let Some(imapresult) = self
             .prepare_imap_operation_on_msg(context, folder, uid)
@@ -197,15 +196,9 @@ impl Imap {
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
 
-        let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
-        uid_message_ids.insert(uid, rfc724_mid);
-        let (last_uid, _received) = match self
-            .fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, false)
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return ImapActionResult::Failed,
-        };
+        let (last_uid, _received) = self
+            .fetch_many_msgs(context, folder, vec![uid], false, false)
+            .await;
         if last_uid.is_none() {
             ImapActionResult::Failed
         } else {
@@ -254,15 +247,13 @@ impl MimeMessage {
 
 #[cfg(test)]
 mod tests {
-    use num_traits::FromPrimitive;
-
+    use super::*;
     use crate::chat::send_msg;
+    use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf_inner;
     use crate::ephemeral::Timer;
-    use crate::message::Viewtype;
     use crate::test_utils::TestContext;
-
-    use super::*;
+    use num_traits::FromPrimitive;
 
     #[test]
     fn test_downloadstate_values() {
@@ -342,15 +333,7 @@ mod tests {
              Date: Sun, 22 Mar 2020 22:37:57 +0000\
              Content-Type: text/plain";
 
-        dc_receive_imf_inner(
-            &t,
-            "Mr.12345678901@example.com",
-            header.as_bytes(),
-            false,
-            Some(100000),
-            false,
-        )
-        .await?;
+        dc_receive_imf_inner(&t, header.as_bytes(), "INBOX", false, Some(100000), false).await?;
         let msg = t.get_last_msg().await;
         assert_eq!(msg.download_state(), DownloadState::Available);
         assert_eq!(msg.get_subject(), "foo");
@@ -361,8 +344,8 @@ mod tests {
 
         dc_receive_imf_inner(
             &t,
-            "Mr.12345678901@example.com",
             format!("{}\n\n100k text...", header).as_bytes(),
+            "INBOX",
             false,
             None,
             false,
@@ -390,7 +373,6 @@ mod tests {
         // download message from bob partially, this must not change the ephemeral timer
         dc_receive_imf_inner(
             &t,
-            "first@example.org",
             b"From: Bob <bob@example.org>\n\
                     To: Alice <alice@example.org>\n\
                     Chat-Version: 1.0\n\
@@ -398,6 +380,7 @@ mod tests {
                     Message-ID: <first@example.org>\n\
                     Date: Sun, 14 Nov 2021 00:10:00 +0000\
                     Content-Type: text/plain",
+            "INBOX",
             false,
             Some(100000),
             false,

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{bail, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
@@ -28,7 +28,13 @@ impl EncryptHelper {
         let prefer_encrypt =
             EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = context.get_primary_self_addr().await?;
+        let addr = match context.get_config(Config::ConfiguredAddr).await? {
+            None => {
+                bail!("addr not configured!");
+            }
+            Some(addr) => addr,
+        };
+
         let public_key = SignedPublicKey::load_self(context).await?;
 
         Ok(EncryptHelper {
@@ -115,9 +121,9 @@ impl EncryptHelper {
             .into_iter()
             .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate
-                .take_key(min_verified)
-                .with_context(|| format!("proper enc-key for {} missing, cannot encrypt", addr))?;
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
+                format_err!("proper enc-key for {} missing, cannot encrypt", addr)
+            })?;
             keyring.add(key);
         }
         keyring.add(self.public_key.clone());
@@ -381,21 +387,30 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 // TODO, remove this once deltachat::key::Key no longer exists.
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .ok_or_else(|| {
+            format_err!(concat!(
+                "Failed to get self address, ",
+                "cannot ensure secret key if not configured."
+            ))
+        })?;
     SignedPublicKey::load_self(context).await?;
     Ok(self_addr)
 }
 
 #[cfg(test)]
 mod tests {
+    use super::*;
+
     use crate::chat;
-    use crate::message::{Message, Viewtype};
+    use crate::constants::Viewtype;
+    use crate::message::Message;
     use crate::param::Param;
     use crate::peerstate::ToSave;
     use crate::test_utils::{bob_keypair, TestContext};
 
-    use super::*;
-
     mod ensure_secret_key_exists {
         use super::*;
 

src/ephemeral.rs

@@ -48,9 +48,9 @@
 //!
 //! ## When messages are deleted
 //!
-//! The `ephemeral_loop` task schedules the next due running of
-//! `delete_expired_messages` which in turn emits `MsgsChanged` events
-//! when deleting local messages to make UIs reload displayed messages.
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
 //!
 //! Server deletion happens by updating the `imap` table based on
 //! the database entries which are expired either according to their
@@ -62,21 +62,19 @@ use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 use anyhow::{ensure, Context as _, Result};
-use async_std::channel::Receiver;
-use async_std::future::timeout;
+use async_std::task;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{send_msg, ChatId};
-use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
-use crate::contact::ContactId;
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
 use crate::context::Context;
-use crate::dc_tools::{duration_to_str, time};
+use crate::dc_tools::time;
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::log::LogExt;
-use crate::message::{Message, MessageState, MsgId, Viewtype};
+use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::sql;
 use crate::stock_str;
 use std::cmp::max;
 
@@ -198,7 +196,7 @@ impl ChatId {
         }
         self.inner_set_ephemeral_timer(context, timer).await?;
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
         msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
         if let Err(err) = send_msg(context, self, &mut msg).await {
             error!(
@@ -214,7 +212,7 @@ impl ChatId {
 pub(crate) async fn stock_ephemeral_timer_changed(
     context: &Context,
     timer: Timer,
-    from_id: ContactId,
+    from_id: u32,
 ) -> String {
     match timer {
         Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
@@ -293,43 +291,12 @@ impl MsgId {
                     paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
-            context.interrupt_ephemeral_task().await;
+            schedule_ephemeral_task(context).await;
         }
         Ok(())
     }
 }
 
-pub(crate) async fn start_ephemeral_timers_msgids(
-    context: &Context,
-    msg_ids: &[MsgId],
-) -> Result<()> {
-    let msg_ids: Vec<&dyn crate::ToSql> = msg_ids
-        .iter()
-        .map(|msg_id| msg_id as &dyn crate::ToSql)
-        .collect();
-    let now = time();
-    let count = context
-        .sql
-        .execute(
-            format!(
-                "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
-         WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
-         AND id IN ({})",
-                sql::repeat_vars(msg_ids.len())
-            ),
-            rusqlite::params_from_iter(
-                std::iter::once(&now as &dyn crate::ToSql)
-                    .chain(std::iter::once(&now as &dyn crate::ToSql))
-                    .chain(msg_ids),
-            ),
-        )
-        .await?;
-    if count > 0 {
-        context.interrupt_ephemeral_task().await;
-    }
-    Ok(())
-}
-
 /// Deletes messages which are expired according to
 /// `delete_device_after` setting or `ephemeral_timestamp` column.
 ///
@@ -338,7 +305,7 @@ pub(crate) async fn start_ephemeral_timers_msgids(
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Result<()> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
     let mut updated = context
         .sql
         .execute(
@@ -354,21 +321,21 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            paramsv![DC_CHAT_ID_TRASH, now, DC_CHAT_ID_TRASH],
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
             .await?
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
             .await?
             .unwrap_or_default();
 
-        let threshold_timestamp = now.saturating_sub(delete_device_after);
+        let threshold_timestamp = time() - delete_device_after;
 
         // Delete expired messages
         //
@@ -378,8 +345,7 @@ WHERE
             .sql
             .execute(
                 "UPDATE msgs \
-             SET chat_id = ?, txt = '', subject='', txt_raw='', \
-                 mime_headers='', from_id=0, to_id=0, param='' \
+             SET txt = 'DELETED', chat_id = ? \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
@@ -398,116 +364,72 @@ WHERE
         updated |= rows_modified > 0;
     }
 
-    if updated {
-        context.emit_msgs_changed_without_ids();
-    }
-
-    Ok(())
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
 }
 
-/// Calculates the next timestamp when a message will be deleted due to
-/// `delete_device_after` setting being set.
-async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
-            .await?
-            .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
-            .await?
-            .unwrap_or_default();
-
-        let oldest_message_timestamp: Option<i64> = context
-            .sql
-            .query_get_value(
-                r#"
-                SELECT min(timestamp)
-                FROM msgs
-                WHERE chat_id > ?
-                  AND chat_id != ?
-                  AND chat_id != ?;
-                "#,
-                paramsv![DC_CHAT_ID_TRASH, self_chat_id, device_chat_id],
-            )
-            .await?;
-
-        Ok(oldest_message_timestamp.map(|x| x.saturating_add(delete_device_after)))
-    } else {
-        Ok(None)
-    }
-}
-
-/// Calculates next timestamp when expiration of some message will happen.
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
 ///
-/// Expiration can happen either because user has set `delete_device_after` setting or because the
-/// message itself has an ephemeral timer.
-async fn next_expiration_timestamp(context: &Context) -> Option<i64> {
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
             r#"
-            SELECT min(ephemeral_timestamp)
-            FROM msgs
-            WHERE ephemeral_timestamp != 0
-              AND chat_id != ?;
-            "#,
+    SELECT ephemeral_timestamp
+    FROM msgs
+    WHERE ephemeral_timestamp != 0
+      AND chat_id != ?
+    ORDER BY ephemeral_timestamp ASC
+    LIMIT 1;
+    "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
         Err(err) => {
             warn!(context, "Can't calculate next ephemeral timeout: {}", err);
-            None
+            return;
         }
         Ok(ephemeral_timestamp) => ephemeral_timestamp,
     };
 
-    let delete_device_after_timestamp: Option<i64> =
-        match next_delete_device_after_timestamp(context).await {
-            Err(err) => {
-                warn!(
-                    context,
-                    "Can't calculate timestamp of the next message expiration: {}", err
-                );
-                None
-            }
-            Ok(timestamp) => timestamp,
-        };
-
-    ephemeral_timestamp
-        .into_iter()
-        .chain(delete_device_after_timestamp.into_iter())
-        .min()
-}
-
-pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiver<()>) {
-    loop {
-        let ephemeral_timestamp = next_expiration_timestamp(context).await;
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
 
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
         let now = SystemTime::now();
-        let until = if let Some(ephemeral_timestamp) = ephemeral_timestamp {
-            UNIX_EPOCH
-                + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
-                + Duration::from_secs(1)
-        } else {
-            // no messages to be deleted for now, wait long for one to occur
-            now + Duration::from_secs(86400)
-        };
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
 
         if let Ok(duration) = until.duration_since(now) {
-            info!(
-                context,
-                "Ephemeral loop waiting for deletion in {} or interrupt",
-                duration_to_str(duration)
-            );
-            if timeout(duration, interrupt_receiver.recv()).await.is_ok() {
-                // received an interruption signal, recompute waiting time (if any)
-                continue;
-            }
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                context1.emit_event(EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0),
+                });
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            context.emit_event(EventType::MsgsChanged {
+                chat_id: ChatId::new(0),
+                msg_id: MsgId::new(0),
+            });
         }
-
-        delete_expired_messages(context, time())
-            .await
-            .ok_or_log(context);
     }
 }
 
@@ -529,11 +451,12 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()
         .execute(
             "UPDATE imap
              SET target=''
-             WHERE rfc724_mid IN (
-               SELECT rfc724_mid FROM msgs
-               WHERE ((download_state = 0 AND timestamp < ?) OR
-                      (download_state != 0 AND timestamp < ?) OR
-                      (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
+             WHERE EXISTS (
+               SELECT * FROM msgs
+               WHERE rfc724_mid=imap.rfc724_mid
+               AND ((download_state = 0 AND timestamp < ?) OR
+                    (download_state != 0 AND timestamp < ?) OR
+                    (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?))
              )",
             paramsv![threshold_timestamp, threshold_timestamp_extended, now],
         )
@@ -577,7 +500,6 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::dc_tools::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
@@ -590,7 +512,7 @@ mod tests {
         let context = TestContext::new().await;
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
             "Message deletion timer is disabled by me."
         );
 
@@ -598,7 +520,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 1 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 s by me."
@@ -607,7 +529,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 30 s by me."
@@ -616,7 +538,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 minute by me."
@@ -625,7 +547,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 90 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1.5 minutes by me."
@@ -634,7 +556,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 30 * 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 30 minutes by me."
@@ -643,7 +565,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 60 * 60 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 hour by me."
@@ -652,7 +574,7 @@ mod tests {
             stock_ephemeral_timer_changed(
                 &context,
                 Timer::Enabled { duration: 5400 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1.5 hours by me."
@@ -663,7 +585,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 2 hours by me."
@@ -674,7 +596,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 day by me."
@@ -685,7 +607,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 2 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 2 days by me."
@@ -696,7 +618,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 1 week by me."
@@ -707,7 +629,7 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                ContactId::SELF
+                DC_CONTACT_ID_SELF
             )
             .await,
             "Message deletion timer is set to 4 weeks by me."
@@ -866,106 +788,31 @@ mod tests {
     #[async_std::test]
     async fn test_ephemeral_delete_msgs() -> Result<()> {
         let t = TestContext::new_alice().await;
-        let self_chat = t.get_self_chat().await;
+        let chat = t.get_self_chat().await;
 
-        assert_eq!(next_expiration_timestamp(&t).await, None);
-
-        t.send_text(self_chat.id, "Saved message, which we delete manually")
+        t.send_text(chat.id, "Saved message, which we delete manually")
             .await;
-        let msg = t.get_last_msg_in(self_chat.id).await;
+        let msg = t.get_last_msg_in(chat.id).await;
         msg.id.delete_from_db(&t).await?;
-        check_msg_is_deleted(&t, &self_chat, msg.id).await;
+        check_msg_was_deleted(&t, &chat, msg.id).await;
 
-        self_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
             .await
             .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
 
-        // Send a saved message which will be deleted after 3600s
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
-            .await
-            .unwrap();
-
-        // Set DeleteDeviceAfter to 1800s. Thend send a saved message which will
-        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
-        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
-            .await
-            .unwrap();
-
-        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
-        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(
-            &t,
-            msg.sender_msg_id,
-            &bob_chat,
-            now + 1799,
-            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
-            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
-            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
-        )
-        .await
-        .unwrap();
-
-        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
-        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
-        bob_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
-            .await
-            .unwrap();
-
-        Ok(())
-    }
-
-    async fn check_msg_will_be_deleted(
-        t: &TestContext,
-        msg_id: MsgId,
-        chat: &Chat,
-        not_deleted_at: i64,
-        deleted_at: i64,
-    ) -> Result<()> {
-        let next_expiration = next_expiration_timestamp(t).await.unwrap();
-
-        assert!(next_expiration > not_deleted_at);
-        delete_expired_messages(t, not_deleted_at).await?;
-
-        let loaded = Message::load_from_db(t, msg_id).await?;
-        assert_eq!(loaded.text.unwrap(), "Message text");
-        assert_eq!(loaded.chat_id, chat.id);
-
-        assert!(next_expiration < deleted_at);
-        delete_expired_messages(t, deleted_at).await?;
-
-        let loaded = Message::load_from_db(t, msg_id).await?;
-        assert_eq!(loaded.text.unwrap(), "");
-        assert_eq!(loaded.chat_id, DC_CHAT_ID_TRASH);
+        async_std::task::sleep(Duration::from_millis(1100)).await;
 
         // Check that the msg was deleted locally.
-        check_msg_is_deleted(t, chat, msg_id).await;
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
 
         Ok(())
     }
 
-    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
         let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
         // Check that the chat is empty except for possibly info messages:
         for item in &chat_items {
@@ -977,8 +824,8 @@ mod tests {
 
         // Check that if there is a message left, the text and metadata are gone
         if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, ContactId::UNDEFINED);
-            assert_eq!(msg.to_id, ContactId::UNDEFINED);
+            assert_eq!(msg.from_id, 0);
+            assert_eq!(msg.to_id, 0);
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
@@ -1115,6 +962,7 @@ mod tests {
                     Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -1135,6 +983,7 @@ mod tests {
                     Ephemeral-Timer: 60\n\
                     \n\
                     second message\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -1171,6 +1020,7 @@ mod tests {
                     In-Reply-To: <first@example.com>\n\
                     \n\
                     > hello\n",
+            "INBOX",
             false,
         )
         .await?;

src/events.rs

@@ -7,10 +7,8 @@ use async_std::path::PathBuf;
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
-use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
-use crate::webxdc::StatusUpdateSerial;
 
 #[derive(Debug)]
 pub struct Events {
@@ -254,7 +252,7 @@ pub enum EventType {
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[strum(props(id = "2030"))]
-    ContactsChanged(Option<ContactId>),
+    ContactsChanged(Option<u32>),
 
     /// Location of one or more contact has changed.
     ///
@@ -262,7 +260,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
     #[strum(props(id = "2035"))]
-    LocationChanged(Option<ContactId>),
+    LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
     #[strum(props(id = "2041"))]
@@ -306,10 +304,7 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[strum(props(id = "2060"))]
-    SecurejoinInviterProgress {
-        contact_id: ContactId,
-        progress: usize,
-    },
+    SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -320,10 +315,7 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
-    SecurejoinJoinerProgress {
-        contact_id: ContactId,
-        progress: usize,
-    },
+    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -334,10 +326,4 @@ pub enum EventType {
 
     #[strum(props(id = "2110"))]
     SelfavatarChanged,
-
-    #[strum(props(id = "2120"))]
-    WebxdcStatusUpdate {
-        msg_id: MsgId,
-        status_update_serial: StatusUpdateSerial,
-    },
 }

src/html.rs

@@ -279,9 +279,9 @@ mod tests {
     use crate::chat;
     use crate::chat::forward_msgs;
     use crate::config::Config;
-    use crate::contact::ContactId;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message::{MessengerMessage, Viewtype};
+    use crate::message::MessengerMessage;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -365,7 +365,7 @@ test some special html-characters as < > and & but also " and &#x
         // however, rust multiline-strings use just `\n`;
         // therefore, we just remove `\r` before comparison.
         assert_eq!(
-            parser.html.replace('\r', ""),
+            parser.html.replace("\r", ""),
             r##"
 <html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
@@ -379,7 +379,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
 </html>
@@ -394,7 +394,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
         let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
-            parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
             r##"<html>
   <p>
     this is <b>html</b>
@@ -440,9 +440,9 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), ContactId::SELF);
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
         assert!(!msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -456,7 +456,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .await
             .unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), ContactId::SELF);
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -469,7 +469,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         let chat = bob.create_chat_with_contact("", "alice@example.org").await;
         bob.recv_msg(&alice.pop_sent_msg().await).await;
         let msg = bob.get_last_msg_in(chat.get_id()).await;
-        assert_ne!(msg.get_from_id(), ContactId::SELF);
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.is_forwarded());
         assert!(msg.get_text().unwrap().contains("this is plain"));
@@ -489,7 +489,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
             .create_chat_with_contact("", "sender@testrun.org")
             .await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        dc_receive_imf(&alice, raw, false).await.unwrap();
+        dc_receive_imf(&alice, raw, "INBOX", false).await.unwrap();
         let msg = alice.get_last_msg_in(chat.get_id()).await;
 
         // forward the message to saved-messages,
@@ -506,7 +506,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         alice.recv_msg(&msg).await;
         let chat = alice.get_self_chat().await;
         let msg = alice.get_last_msg_in(chat.get_id()).await;
-        assert_eq!(msg.get_from_id(), ContactId::SELF);
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
         assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
         assert!(msg.get_showpadlock());
         assert!(msg.is_forwarded());
@@ -555,6 +555,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         dc_receive_imf(
             &t,
             include_bytes!("../test-data/message/cp1252-html.eml"),
+            "INBOX",
             false,
         )
         .await?;

src/imap/idle.rs

@@ -1,6 +1,6 @@
 use super::Imap;
 
-use anyhow::{bail, Context as _, Result};
+use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
@@ -31,7 +31,7 @@ impl Imap {
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if self.server_sent_unsolicited_exists(context)? {
+        if self.server_sent_unsolicited_exists(context) {
             return Ok(info);
         }
 
@@ -90,8 +90,8 @@ impl Imap {
             let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
-                .await?
-                .context("IMAP IDLE protocol timed out")?;
+                .await
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
             self.session = Some(Session { inner: session });
         } else {
             warn!(context, "Attempted to idle without a session");
@@ -157,10 +157,8 @@ impl Imap {
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-                    match self
-                        .fetch_new_messages(context, &watch_folder, false, false)
-                        .await
-                    {
+
+                    match self.fetch_new_messages(context, &watch_folder, false).await {
                         Ok(res) => {
                             info!(context, "fetch_new_messages returned {:?}", res);
                             if res {

src/imap/scan_folders.rs

@@ -2,18 +2,15 @@ use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
 
-use crate::config::Config;
 use crate::imap::Imap;
-use crate::log::LogExt;
+use crate::{config::Config, log::LogExt};
 use crate::{context::Context, imap::FolderMeaning};
-
-use async_std::stream::StreamExt;
+use async_std::prelude::*;
 
 use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
-    /// Returns true if folders were scanned, false if scanning was postponed.
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
+    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<()> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -23,18 +20,28 @@ impl Imap {
                 .await?;
 
             if elapsed_secs < debounce_secs {
-                return Ok(false);
+                return Ok(());
             }
         }
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let folders = self.list_folders(context).await?;
+        let session = self.session.as_mut();
+        let session = session.context("scan_folders(): IMAP No Connection established")?;
+        let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
 
         for folder in folders {
+            let folder = match folder {
+                Ok(f) => f,
+                Err(e) => {
+                    warn!(context, "Can't get folder: {}", e);
+                    continue;
+                }
+            };
+
             // Gmail labels are not folders and should be skipped. For example,
             // emails appear in the inbox and under "All Mail" as soon as it is
             // received. The code used to wrongly conclude that the email had
@@ -60,73 +67,55 @@ impl Imap {
             let is_drafts = folder_meaning == FolderMeaning::Drafts
                 || (folder_meaning == FolderMeaning::Unknown
                     && folder_name_meaning == FolderMeaning::Drafts);
-            let is_spam_folder = folder_meaning == FolderMeaning::Spam
-                || (folder_meaning == FolderMeaning::Unknown
-                    && folder_name_meaning == FolderMeaning::Spam);
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
                 // Drain leftover unsolicited EXISTS messages
-                self.server_sent_unsolicited_exists(context)?;
+                self.server_sent_unsolicited_exists(context);
 
                 loop {
-                    self.fetch_move_delete(context, folder.name(), is_spam_folder)
+                    self.fetch_move_delete(context, folder.name())
                         .await
                         .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
 
                     // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
-                    if !self.server_sent_unsolicited_exists(context)? {
+                    if !self.server_sent_unsolicited_exists(context) {
                         break;
                     }
                 }
             }
         }
 
-        // Set the `ConfiguredSentboxFolder` or set it to `None` if the folder was deleted.
-        context
-            .set_config(
-                Config::ConfiguredSentboxFolder,
-                folder_configs
-                    .get(&Config::ConfiguredSentboxFolder)
-                    .map(|s| s.as_str()),
-            )
-            .await?;
+        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
+        // `ConfiguredSentboxFolder` is set to `None`:
+        for config in &[
+            Config::ConfiguredSentboxFolder,
+            Config::ConfiguredSpamFolder,
+        ] {
+            context
+                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
+                .await?;
+        }
 
         last_scan.replace(Instant::now());
-        Ok(true)
+        Ok(())
     }
-
-    /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(
-        self: &mut Imap,
-        context: &Context,
-    ) -> Result<Vec<async_imap::types::Name>> {
-        let session = self.session.as_mut();
-        let session = session.context("No IMAP connection")?;
-        let list = session
-            .list(Some(""), Some("*"))
-            .await?
-            .filter_map(|f| f.ok_or_log_msg(context, "list_folders() can't get folder"));
-        Ok(list.collect().await)
-    }
-}
-
-pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {
-    let mut res = vec![Config::ConfiguredInboxFolder];
-    if context.get_config_bool(Config::SentboxWatch).await? {
-        res.push(Config::ConfiguredSentboxFolder);
-    }
-    if context.should_watch_mvbox().await? {
-        res.push(Config::ConfiguredMvboxFolder);
-    }
-    Ok(res)
 }
 
 pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
     let mut res = Vec::new();
-    for folder_config in get_watched_folder_configs(context).await? {
-        if let Some(folder) = context.get_config(folder_config).await? {
-            res.push(folder);
+    if let Some(inbox_folder) = context.get_config(Config::ConfiguredInboxFolder).await? {
+        res.push(inbox_folder);
+    }
+    let folder_watched_configured = &[
+        (Config::SentboxWatch, Config::ConfiguredSentboxFolder),
+        (Config::MvboxMove, Config::ConfiguredMvboxFolder),
+    ];
+    for (watched, configured) in folder_watched_configured {
+        if context.get_config_bool(*watched).await? {
+            if let Some(folder) = context.get_config(*configured).await? {
+                res.push(folder);
+            }
         }
     }
     Ok(res)

src/imap/select_folder.rs

@@ -93,11 +93,7 @@ impl Imap {
         // select new folder
         if let Some(folder) = folder {
             if let Some(ref mut session) = &mut self.session {
-                let res = if self.config.can_condstore {
-                    session.select_condstore(folder).await
-                } else {
-                    session.select(folder).await
-                };
+                let res = session.select(folder).await;
 
                 // <https://tools.ietf.org/html/rfc3501#section-6.3.1>
                 // says that if the server reports select failure we are in

src/imex.rs

@@ -16,21 +16,21 @@ use rand::{thread_rng, Rng};
 use crate::blob::BlobObject;
 use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
-use crate::contact::ContactId;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
 use crate::dc_tools::{
-    dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
-    dc_open_file_std, dc_read_file, dc_write_file, time, EmailAddress,
+    dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc,
+    dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress,
 };
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::log::LogExt;
-use crate::message::{Message, MsgId, Viewtype};
+use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
-use crate::sql;
+use crate::sql::{self, Sql};
 use crate::stock_str;
 
 // Name of the database file in the backup.
@@ -41,24 +41,24 @@ const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 #[repr(u32)]
 pub enum ImexMode {
     /// Export all private keys and all public keys of the user to the
-    /// directory given as `path`.  The default key is written to the files `public-key-default.asc`
+    /// directory given as `param1`.  The default key is written to the files `public-key-default.asc`
     /// and `private-key-default.asc`, if there are more keys, they are written to files as
     /// `public-key-<id>.asc` and `private-key-<id>.asc`
     ExportSelfKeys = 1,
 
-    /// Import private keys found in the directory given as `path`.
+    /// Import private keys found in the directory given as `param1`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
 
-    /// Export a backup to the directory given as `path` with the given `passphrase`.
+    /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
     /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
     /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
-    /// `path` is the file (not: directory) to import. The file is normally
+    /// `param1` is the file (not: directory) to import. The file is normally
     /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
     ImportBackup = 12,
@@ -78,16 +78,11 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(
-    context: &Context,
-    what: ImexMode,
-    path: &Path,
-    passphrase: Option<String>,
-) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: &Path) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
-        let success = imex_inner(context, what, path, passphrase).await;
+        let success = imex_inner(context, what, param1).await;
         match success {
             Ok(()) => {
                 info!(context, "IMEX successfully completed");
@@ -120,10 +115,15 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
         dc_delete_file(context, context.get_dbfile()).await;
         dc_delete_files_in_dir(context, context.get_blobdir()).await;
     }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
+    }
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
+pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
@@ -145,6 +145,59 @@ pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result<String> {
         }
     }
 
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String> {
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_time = 0;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name = name.to_string_lossy();
+            if name.starts_with("delta-chat") && name.ends_with(".bak") {
+                let sql = Sql::new();
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int("backup_time")
+                            .await?
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
+                    }
+                    Err(e) => {
+                        warn!(
+                            context,
+                            "Found backup file {} which could not be opened: {}", name, e
+                        );
+                        // On some Android devices we can't open sql files that are not in our private directory
+                        // (see <https://github.com/deltachat/deltachat-android/issues/1768>). So, compare names
+                        // to still find the newest backup.
+                        let name: String = name.into();
+                        if newest_backup_time == 0
+                            && (newest_backup_name.is_empty() || name > newest_backup_name)
+                        {
+                            newest_backup_path = Some(path);
+                            newest_backup_name = name;
+                        }
+                    }
+                }
+            }
+        }
+    }
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
         None => bail!("no backup found in {}", dir_name.display()),
@@ -165,6 +218,7 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
 }
 
 async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
+    let mut msg: Message;
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
     let setup_file_content = render_setup_file(context, &setup_code).await?;
@@ -176,11 +230,9 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
-    let mut msg = Message {
-        viewtype: Viewtype::File,
-        ..Default::default()
-    };
+    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
+    msg = Message::default();
+    msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
     msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
@@ -237,7 +289,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
 
     let msg_subj = stock_str::ac_setup_msg_subject(context).await;
     let msg_body = stock_str::ac_setup_msg_body(context).await;
-    let msg_body_html = msg_body.replace('\r', "").replace('\n', "<br>");
+    let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
     Ok(format!(
         concat!(
             "<!DOCTYPE html>\r\n",
@@ -351,8 +403,9 @@ async fn set_self_key(
         }
     };
 
-    let self_addr = context.get_primary_self_addr().await?;
-    let addr = EmailAddress::new(&self_addr)?;
+    let self_addr = context.get_config(Config::ConfiguredAddr).await?;
+    ensure!(self_addr.is_some(), "Missing self addr");
+    let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
     let keypair = pgp::KeyPair {
         addr,
         public: public_key,
@@ -396,12 +449,7 @@ fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(
-    context: &Context,
-    what: ImexMode,
-    path: &Path,
-    passphrase: Option<String>,
-) -> Result<()> {
+async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()> {
     info!(context, "Import/export dir: {}", path.display());
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
@@ -419,27 +467,26 @@ async fn imex_inner(
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
 
-        ImexMode::ExportBackup => {
-            export_backup(context, path, passphrase.unwrap_or_default()).await
-        }
-        ImexMode::ImportBackup => {
-            import_backup(context, path, passphrase.unwrap_or_default()).await?;
-            context.sql.run_migrations(context).await
-        }
+        ImexMode::ExportBackup => export_backup(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
+        ImexMode::ImportBackup => import_backup(context, path).await,
     }
 }
 
-/// Imports backup into the currently open database.
-///
-/// The contents of the currently open database will be lost.
-///
-/// `passphrase` is the passphrase used to open backup database. If backup is unencrypted, pass
-/// empty string here.
-async fn import_backup(
-    context: &Context,
-    backup_to_import: &Path,
-    passphrase: String,
-) -> Result<()> {
+/// Import Backup
+async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> {
+    if backup_to_import.to_string_lossy().ends_with(".bak") {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.display(),
+        context.get_dbfile().display()
+    );
+
     ensure!(
         !context.is_configured().await?,
         "Cannot import backups to accounts in use."
@@ -448,19 +495,15 @@ async fn import_backup(
         !context.scheduler.read().await.is_running(),
         "cannot import backup, IO already running"
     );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
 
     let backup_file = File::open(backup_to_import).await?;
     let file_size = backup_file.metadata().await?.len();
-    info!(
-        context,
-        "Import \"{}\" ({} bytes) to \"{}\".",
-        backup_to_import.display(),
-        file_size,
-        context.get_dbfile().display()
-    );
-
-    context.sql.config_cache.write().await.clear();
-
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
@@ -479,15 +522,11 @@ async fn import_backup(
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
             // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
             f.unpack_in(context.get_blobdir()).await?;
-            let unpacked_database = context.get_blobdir().join(DBFILE_BACKUP_NAME);
-            context
-                .sql
-                .import(&unpacked_database, passphrase.clone())
-                .await
-                .context("cannot import unpacked database")?;
-            fs::remove_file(unpacked_database)
-                .await
-                .context("cannot remove unpacked database")?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
         } else {
             // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
             f.unpack_in(context.get_blobdir()).await?;
@@ -502,52 +541,136 @@ async fn import_backup(
         }
     }
 
+    context
+        .sql
+        .open(context, context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
     delete_and_reset_all_device_msgs(context).await?;
 
     Ok(())
 }
 
+async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> {
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await?,
+        "Cannot import backups to accounts in use."
+    );
+    ensure!(
+        !context.scheduler.read().await.is_running(),
+        "cannot import backup, IO already running"
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    ensure!(
+        dc_copy_file(context, backup_to_import, context.get_dbfile()).await,
+        "could not copy file"
+    );
+    /* error already logged */
+    /* re-open copied database file */
+    context
+        .sql
+        .open(context, context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
+    delete_and_reset_all_device_msgs(context).await?;
+
+    let total_files_cnt = context
+        .sql
+        .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![])
+        .await?;
+    info!(
+        context,
+        "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
+    );
+
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
+        .sql
+        .query_map(
+            "SELECT id FROM backup_blobs ORDER BY id",
+            paramsv![],
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?;
+
+    let mut all_files_extracted = true;
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| {
+                    let file_name: String = row.get(0)?;
+                    let file_blob: Vec<u8> = row.get(1)?;
+                    Ok((file_name, file_blob))
+                },
+            )
+            .await?;
+
+        if context.shall_stop_ongoing().await {
+            all_files_extracted = false;
+            break;
+        }
+        let mut permille = processed_files_cnt * 1000 / total_files_cnt;
+        if permille < 10 {
+            permille = 10
+        }
+        if permille > 990 {
+            permille = 990
+        }
+        context.emit_event(EventType::ImexProgress(permille));
+        if file_blob.is_empty() {
+            continue;
+        }
+
+        let path_filename = context.get_blobdir().join(file_name);
+        dc_write_file(context, &path_filename, &file_blob).await?;
+    }
+
+    if all_files_extracted {
+        // only delete backup_blobs if all files were successfully extracted
+        context
+            .sql
+            .execute("DROP TABLE backup_blobs;", paramsv![])
+            .await?;
+        context.sql.execute("VACUUM;", paramsv![]).await.ok();
+        Ok(())
+    } else {
+        bail!("received stop signal");
+    }
+}
+
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-
-/// Returns Ok((temp_db_path, temp_path, dest_path)) on success. Unencrypted database can be
-/// written to temp_db_path. The backup can then be written to temp_path. If the backup succeeded,
-/// it can be renamed to dest_path. This guarantees that the backup is complete.
-async fn get_next_backup_path(
-    folder: &Path,
-    backup_time: i64,
-) -> Result<(PathBuf, PathBuf, PathBuf)> {
-    let folder = PathBuf::from(folder);
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
-        .format("delta-chat-backup-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut tempdbfile = folder.clone();
-        tempdbfile.push(format!("{}-{:02}.db", stem, i));
-
-        let mut tempfile = folder.clone();
-        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
-
-        let mut destfile = folder.clone();
-        destfile.push(format!("{}-{:02}.tar", stem, i));
-
-        if !tempdbfile.exists().await && !tempfile.exists().await && !destfile.exists().await {
-            return Ok((tempdbfile, tempfile, destfile));
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
-async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
+#[allow(unused)]
+async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
-    let (temp_db_path, temp_path, dest_path) = get_next_backup_path(dir, now).await?;
-    let _d1 = DeleteOnDrop(temp_db_path.clone());
-    let _d2 = DeleteOnDrop(temp_path.clone());
+    let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
 
     context
         .sql
@@ -559,14 +682,16 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
         .sql
         .execute("VACUUM;", paramsv![])
         .await
-        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
-        .ok();
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
     ensure!(
         !context.scheduler.read().await.is_running(),
         "cannot export backup, IO already running"
     );
 
+    // we close the database during the export
+    context.sql.close().await;
+
     info!(
         context,
         "Backup '{}' to '{}'.",
@@ -574,13 +699,10 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
         dest_path.display(),
     );
 
-    context
-        .sql
-        .export(&temp_db_path, passphrase)
-        .await
-        .with_context(|| format!("failed to backup plaintext database to {:?}", temp_db_path))?;
+    let res = export_backup_inner(context, &temp_path).await;
 
-    let res = export_backup_inner(context, &temp_db_path, &temp_path).await;
+    // we re-open the database after export is finished
+    context.sql.open(context, context.get_dbfile(), false).await;
 
     match &res {
         Ok(_) => {
@@ -599,21 +721,18 @@ impl Drop for DeleteOnDrop {
     fn drop(&mut self) {
         let file = self.0.clone();
         // Not using dc_delete_file() here because it would send a DeletedBlobFile event
-        async_std::task::block_on(fs::remove_file(file)).ok();
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
     }
 }
 
-async fn export_backup_inner(
-    context: &Context,
-    temp_db_path: &Path,
-    temp_path: &Path,
-) -> Result<()> {
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
     let file = File::create(temp_path).await?;
 
     let mut builder = async_tar::Builder::new(file);
 
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
     builder
-        .append_path_with_name(temp_db_path, DBFILE_BACKUP_NAME)
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
         .await?;
 
     let read_dir: Vec<_> = fs::read_dir(context.get_blobdir()).await?.collect().await;
@@ -892,64 +1011,16 @@ mod tests {
     async fn test_export_and_import_key() {
         let context = TestContext::new_alice().await;
         let blobdir = context.ctx.get_blobdir();
-        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
             panic!("got error on export: {:?}", err);
         }
 
         let context2 = TestContext::new_alice().await;
-        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir).await {
             panic!("got error on import: {:?}", err);
         }
     }
 
-    #[async_std::test]
-    async fn test_export_and_import_backup() -> Result<()> {
-        let backup_dir = tempfile::tempdir().unwrap();
-
-        let context1 = TestContext::new_alice().await;
-        assert!(context1.is_configured().await?);
-
-        let context2 = TestContext::new().await;
-        assert!(!context2.is_configured().await?);
-        assert!(has_backup(&context2, backup_dir.path().as_ref())
-            .await
-            .is_err());
-
-        // export from context1
-        assert!(imex(
-            &context1,
-            ImexMode::ExportBackup,
-            backup_dir.path().as_ref(),
-            None,
-        )
-        .await
-        .is_ok());
-        let _event = context1
-            .evtracker
-            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
-            .await;
-
-        // import to context2
-        let backup = has_backup(&context2, backup_dir.path().as_ref()).await?;
-        assert!(
-            imex(&context2, ImexMode::ImportBackup, backup.as_ref(), None)
-                .await
-                .is_ok()
-        );
-        let _event = context2
-            .evtracker
-            .get_matching(|evt| matches!(evt, EventType::ImexProgress(1000)))
-            .await;
-
-        assert!(context2.is_configured().await?);
-        assert_eq!(
-            context2.get_config(Config::Addr).await?,
-            Some("alice@example.org".to_string())
-        );
-
-        Ok(())
-    }
-
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");

src/job.rs

@@ -3,22 +3,28 @@
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
 use std::fmt;
+use std::future::Future;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
+use async_smtp::smtp::response::{Category, Code, Detail};
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
+use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::contact::{normalize_name, Contact, ContactId, Modifier, Origin};
+use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
-use crate::dc_tools::time;
+use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::events::EventType;
-use crate::imap::Imap;
-use crate::message::{Message, MsgId};
+use crate::imap::{Imap, ImapActionResult};
+use crate::location;
+use crate::log::LogExt;
+use crate::message::{self, Message, MessageState, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
 use crate::scheduler::InterruptInfo;
-use crate::smtp::{smtp_send, SendResult, Smtp};
+use crate::smtp::Smtp;
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -30,6 +36,7 @@ const JOB_RETRIES: u32 = 17;
 )]
 #[repr(u32)]
 pub(crate) enum Thread {
+    Unknown = 0,
     Imap = 100,
     Smtp = 5000,
 }
@@ -37,7 +44,7 @@ pub(crate) enum Thread {
 /// Job try result.
 #[derive(Debug, Display)]
 pub enum Status {
-    Finished(Result<()>),
+    Finished(std::result::Result<(), Error>),
     RetryNow,
     RetryLater,
 }
@@ -57,6 +64,12 @@ macro_rules! job_try {
     };
 }
 
+impl Default for Thread {
+    fn default() -> Self {
+        Thread::Unknown
+    }
+}
+
 #[derive(
     Debug,
     Display,
@@ -72,8 +85,12 @@ macro_rules! job_try {
 )]
 #[repr(u32)]
 pub enum Action {
+    Unknown = 0,
+
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
+    Housekeeping = 105, // low priority ...
     FetchExistingMsgs = 110,
+    MarkseenMsgOnImap = 130,
 
     // this is user initiated so it should have a fairly high priority
     UpdateRecentQuota = 140,
@@ -89,7 +106,16 @@ pub enum Action {
     ResyncFolders = 300,
 
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
+    MaybeSendLocations = 5005, // low priority ...
+    MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
+    SendMsgToSmtp = 5901, // ... high priority
+}
+
+impl Default for Action {
+    fn default() -> Self {
+        Action::Unknown
+    }
 }
 
 impl From<Action> for Thread {
@@ -97,12 +123,19 @@ impl From<Action> for Thread {
         use Action::*;
 
         match action {
+            Unknown => Thread::Unknown,
+
+            Housekeeping => Thread::Imap,
             FetchExistingMsgs => Thread::Imap,
             ResyncFolders => Thread::Imap,
+            MarkseenMsgOnImap => Thread::Imap,
             UpdateRecentQuota => Thread::Imap,
             DownloadMsg => Thread::Imap,
 
+            MaybeSendLocations => Thread::Smtp,
+            MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
+            SendMsgToSmtp => Thread::Smtp,
         }
     }
 }
@@ -170,7 +203,7 @@ impl Job {
                     "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
                     paramsv![
                         self.desired_timestamp,
-                        i64::from(self.tries),
+                        self.tries as i64,
                         self.param.to_string(),
                         self.job_id as i32,
                     ],
@@ -193,11 +226,222 @@ impl Job {
         Ok(())
     }
 
+    async fn smtp_send<F, Fut>(
+        &mut self,
+        context: &Context,
+        recipients: Vec<async_smtp::EmailAddress>,
+        message: Vec<u8>,
+        job_id: u32,
+        smtp: &mut Smtp,
+        success_cb: F,
+    ) -> Status
+    where
+        F: FnOnce() -> Fut,
+        Fut: Future<Output = Result<()>>,
+    {
+        // hold the smtp lock during sending of a job and
+        // its ok/error response processing. Note that if a message
+        // was sent we need to mark it in the database ASAP as we
+        // otherwise might send it twice.
+        if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+            info!(context, "smtp-sending out mime message:");
+            println!("{}", String::from_utf8_lossy(&message));
+        }
+
+        smtp.connectivity.set_working(context).await;
+
+        let send_result = smtp.send(context, recipients, message, job_id).await;
+        smtp.last_send_error = send_result.as_ref().err().map(|e| e.to_string());
+
+        let status = match send_result {
+            Err(crate::smtp::send::Error::SmtpSend(err)) => {
+                // Remote error, retry later.
+                warn!(context, "SMTP failed to send: {:?}", &err);
+
+                let res = match err {
+                    async_smtp::smtp::error::Error::Permanent(ref response) => {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
+                            // Sometimes servers send a permanent error when actually it is a temporary error
+                            // For documentation see <https://tools.ietf.org/html/rfc3463>
+                            Code {
+                                category: Category::MailSystem,
+                                detail: Detail::Zero,
+                                ..
+                            } => {
+                                // Ignore status code 5.5.0, see <https://support.delta.chat/t/every-other-message-gets-stuck/877/2>
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.first_word() == Some(&"5.5.0".to_string())
+                            }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                        }
+                    }
+                    async_smtp::smtp::error::Error::Transient(ref response) => {
+                        // We got a transient 4xx response from SMTP server.
+                        // Give some time until the server-side error maybe goes away.
+
+                        if let Some(first_word) = response.first_word() {
+                            if first_word.ends_with(".1.1")
+                                || first_word.ends_with(".1.2")
+                                || first_word.ends_with(".1.3")
+                            {
+                                // Sometimes we receive transient errors that should be permanent.
+                                // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
+                                // receive as a transient error are misconfigurations of the smtp server.
+                                // See <https://tools.ietf.org/html/rfc3463#section-3.2>
+                                info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
+                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } else {
+                                Status::RetryLater
+                            }
+                        } else {
+                            Status::RetryLater
+                        }
+                    }
+                    _ => {
+                        if smtp.has_maybe_stale_connection().await {
+                            info!(context, "stale connection? immediately reconnecting");
+                            Status::RetryNow
+                        } else {
+                            Status::RetryLater
+                        }
+                    }
+                };
+
+                // this clears last_success info
+                smtp.disconnect().await;
+
+                res
+            }
+            Err(crate::smtp::send::Error::Envelope(err)) => {
+                // Local error, job is invalid, do not retry.
+                smtp.disconnect().await;
+                warn!(context, "SMTP job is invalid: {}", err);
+                Status::Finished(Err(err.into()))
+            }
+            Err(crate::smtp::send::Error::NoTransport) => {
+                // Should never happen.
+                // It does not even make sense to disconnect here.
+                error!(context, "SMTP job failed because SMTP has no transport");
+                Status::Finished(Err(format_err!("SMTP has not transport")))
+            }
+            Err(crate::smtp::send::Error::Other(err)) => {
+                // Local error, job is invalid, do not retry.
+                smtp.disconnect().await;
+                warn!(context, "unable to load job: {}", err);
+                Status::Finished(Err(err))
+            }
+            Ok(()) => {
+                job_try!(success_cb().await);
+                Status::Finished(Ok(()))
+            }
+        };
+
+        if let Status::Finished(Err(err)) = &status {
+            // We couldn't send the message, so mark it as failed
+            let msg_id = MsgId::new(self.foreign_id);
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+        }
+        status
+    }
+
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+        //  SMTP server, if not yet done
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            smtp.last_send_error = Some(format!("SMTP connection failure: {:#}", err));
+            return Status::RetryLater;
+        }
+
+        let filename = job_try!(job_try!(self
+            .param
+            .get_path(Param::File, context)
+            .map_err(|_| format_err!("Can't get filename")))
+        .ok_or_else(|| format_err!("Can't get filename")));
+        let body = job_try!(dc_read_file(context, &filename).await);
+        let recipients = job_try!(self.param.get(Param::Recipients).ok_or_else(|| {
+            warn!(context, "Missing recipients for job {}", self.job_id);
+            format_err!("Missing recipients")
+        }));
+
+        let recipients_list = recipients
+            .split('\x1e')
+            .filter_map(
+                |addr| match async_smtp::EmailAddress::new(addr.to_string()) {
+                    Ok(addr) => Some(addr),
+                    Err(err) => {
+                        warn!(context, "invalid recipient: {} {:?}", addr, err);
+                        None
+                    }
+                },
+            )
+            .collect::<Vec<_>>();
+
+        /* if there is a msg-id and it does not exist in the db, cancel sending.
+        this happends if dc_delete_msgs() was called
+        before the generated mime was sent out */
+        if 0 != self.foreign_id {
+            match message::exists(context, MsgId::new(self.foreign_id)).await {
+                Ok(exists) => {
+                    if !exists {
+                        return Status::Finished(Err(format_err!(
+                            "Not sending Message {} as it was deleted",
+                            self.foreign_id
+                        )));
+                    }
+                }
+                Err(err) => {
+                    warn!(context, "failed to check message existence: {:?}", err);
+                    smtp.last_send_error =
+                        Some(format!("failed to check message existence: {:#}", err));
+                    return Status::RetryLater;
+                }
+            }
+        };
+
+        let foreign_id = self.foreign_id;
+        self.smtp_send(context, recipients_list, body, self.job_id, smtp, || {
+            async move {
+                // smtp success, update db ASAP, then delete smtp file
+                if 0 != foreign_id {
+                    set_delivered(context, MsgId::new(foreign_id)).await?;
+                }
+                // now also delete the generated file
+                dc_delete_file(context, filename).await;
+
+                // finally, create another send-job if there are items to be synced.
+                // triggering sync-job after msg-send-job guarantees, the recipient has grpid etc.
+                // once the sync message arrives.
+                // if there are no items to sync, this function returns fast.
+                context.send_sync_msg().await?;
+
+                Ok(())
+            }
+        })
+        .await
+    }
+
     /// Get `SendMdn` jobs with foreign_id equal to `contact_id` excluding the `job_id` job.
     async fn get_additional_mdn_jobs(
         &self,
         context: &Context,
-        contact_id: ContactId,
+        contact_id: u32,
     ) -> Result<(Vec<u32>, Vec<String>)> {
         // Extract message IDs from job parameters
         let res: Vec<(u32, MsgId)> = context
@@ -244,7 +488,7 @@ impl Job {
             return Status::Finished(Err(format_err!("MDNs are disabled")));
         }
 
-        let contact_id = ContactId::new(self.foreign_id);
+        let contact_id = self.foreign_id;
         let contact = job_try!(Contact::load_from_db(context, contact_id).await);
         if contact.is_blocked() {
             return Status::Finished(Err(format_err!("Contact is blocked")));
@@ -291,18 +535,14 @@ impl Job {
             return Status::RetryLater;
         }
 
-        match smtp_send(context, &recipients, &body, smtp, msg_id, 0).await {
-            SendResult::Success => {
+        self.smtp_send(context, recipients, body, self.job_id, smtp, || {
+            async move {
                 // Remove additional SendMdn jobs we have aggregated into this one.
-                job_try!(kill_ids(context, &additional_job_ids).await);
-                Status::Finished(Ok(()))
+                kill_ids(context, &additional_job_ids).await?;
+                Ok(())
             }
-            SendResult::Retry => {
-                info!(context, "Temporary SMTP failure while sending an MDN");
-                Status::RetryLater
-            }
-            SendResult::Failure(err) => Status::Finished(Err(err)),
-        }
+        })
+        .await
     }
 
     /// Read the recipients from old emails sent by the user and add them as contacts.
@@ -330,7 +570,7 @@ impl Job {
                 Config::ConfiguredSentboxFolder,
             ] {
                 if let Some(folder) = job_try!(context.get_config(*config).await) {
-                    if let Err(e) = imap.fetch_new_messages(context, &folder, false, true).await {
+                    if let Err(e) = imap.fetch_new_messages(context, &folder, true).await {
                         // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
                         warn!(context, "Could not fetch messages, retrying: {:#}", e);
                         return Status::RetryLater;
@@ -343,38 +583,98 @@ impl Job {
         Status::Finished(Ok(()))
     }
 
-    /// Synchronizes UIDs for all folders.
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
 
-        let all_folders = match imap.list_folders(context).await {
-            Ok(v) => v,
-            Err(e) => {
-                warn!(context, "Listing folders for resync failed: {:#}", e);
-                return Status::RetryLater;
-            }
-        };
+        let sentbox_folder = job_try!(context.get_config(Config::ConfiguredSentboxFolder).await);
+        if let Some(sentbox_folder) = sentbox_folder {
+            job_try!(imap.resync_folder_uids(context, sentbox_folder).await);
+        }
 
-        let mut any_failed = false;
+        let inbox_folder = job_try!(context.get_config(Config::ConfiguredInboxFolder).await);
+        if let Some(inbox_folder) = inbox_folder {
+            job_try!(imap.resync_folder_uids(context, inbox_folder).await);
+        }
 
-        for folder in all_folders {
-            if let Err(e) = imap
-                .resync_folder_uids(context, folder.name().to_string())
+        let mvbox_folder = job_try!(context.get_config(Config::ConfiguredMvboxFolder).await);
+        if let Some(mvbox_folder) = mvbox_folder {
+            job_try!(imap.resync_folder_uids(context, mvbox_folder).await);
+        }
+
+        Status::Finished(Ok(()))
+    }
+
+    async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.prepare(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let row = job_try!(
+            context
+                .sql
+                .query_row_optional(
+                    "SELECT uid, folder FROM imap
+                    WHERE rfc724_mid=? AND folder=target
+                    ORDER BY uid ASC
+                    LIMIT 1",
+                    paramsv![msg.rfc724_mid],
+                    |row| {
+                        let uid: u32 = row.get(0)?;
+                        let folder: String = row.get(1)?;
+                        Ok((uid, folder))
+                    }
+                )
                 .await
-            {
-                warn!(context, "{:#}", e);
-                any_failed = true;
+        );
+        if let Some((server_uid, server_folder)) = row {
+            let result = imap.set_seen(context, &server_folder, server_uid).await;
+            match result {
+                ImapActionResult::RetryLater => return Status::RetryLater,
+                ImapActionResult::Success | ImapActionResult::Failed => {}
+            }
+        } else {
+            info!(
+                context,
+                "Can't mark the message {} as seen on IMAP because there is no known UID",
+                msg.rfc724_mid
+            );
+        }
+
+        // XXX we send MDN even in case of failure to mark the messages as seen, e.g. if it was
+        // already deleted on the server by another device. The job will not be retried so locally
+        // there is no risk of double-sending MDNs.
+        //
+        // Read receipts for system messages are never sent. These messages have no place to
+        // display received read receipt anyway.  And since their text is locally generated,
+        // quoting them is dangerous as it may contain contact names. E.g., for original message
+        // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
+        // be a display name stored in address book rather than the name sent in the From field by
+        // the user.
+        if msg.param.get_bool(Param::WantsMdn).unwrap_or_default() && !msg.is_system_message() {
+            let mdns_enabled = job_try!(context.get_config_bool(Config::MdnsEnabled).await);
+            if mdns_enabled {
+                if let Err(err) = send_mdn(context, &msg).await {
+                    warn!(context, "could not send out mdn for {}: {}", msg.id, err);
+                    return Status::Finished(Err(err));
+                }
             }
         }
 
-        if any_failed {
-            Status::RetryLater
-        } else {
-            Status::Finished(Ok(()))
-        }
+        Status::Finished(Ok(()))
     }
 }
 
@@ -389,12 +689,9 @@ pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
 
 /// Remove jobs with specified IDs.
 async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
-    if job_ids.is_empty() {
-        return Ok(());
-    }
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
-        sql::repeat_vars(job_ids.len())
+        job_ids.iter().map(|_| "?").collect::<Vec<&str>>().join(",")
     );
     context
         .sql
@@ -414,6 +711,17 @@ pub async fn action_exists(context: &Context, action: Action) -> Result<bool> {
     Ok(exists)
 }
 
+async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
+    message::update_msg_state(context, msg_id, MessageState::OutDelivered).await;
+    let chat_id: ChatId = context
+        .sql
+        .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![msg_id])
+        .await?
+        .unwrap_or_default();
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
+    Ok(())
+}
+
 async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
     let mailbox = if let Ok(Some(m)) = context.get_config(folder).await {
         m
@@ -459,6 +767,132 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
     };
 }
 
+/// Constructs a job for sending a message.
+///
+/// Returns `None` if no messages need to be sent out.
+///
+/// In order to be processed, must be `add`ded.
+pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job>> {
+    let mut msg = Message::load_from_db(context, msg_id).await?;
+    msg.try_calc_and_set_dimensions(context).await.ok();
+
+    /* create message */
+    let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
+
+    let attach_selfavatar = match chat::shall_attach_selfavatar(context, msg.chat_id).await {
+        Ok(attach_selfavatar) => attach_selfavatar,
+        Err(err) => {
+            warn!(context, "job: cannot get selfavatar-state: {}", err);
+            false
+        }
+    };
+
+    let mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+
+    let mut recipients = mimefactory.recipients();
+
+    let from = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .unwrap_or_default();
+    let lowercase_from = from.to_lowercase();
+
+    // Send BCC to self if it is enabled and we are not going to
+    // delete it immediately.
+    if context.get_config_bool(Config::BccSelf).await?
+        && context.get_config_delete_server_after().await? != Some(0)
+        && !recipients
+            .iter()
+            .any(|x| x.to_lowercase() == lowercase_from)
+    {
+        recipients.push(from);
+    }
+
+    if recipients.is_empty() {
+        // may happen eg. for groups with only SELF and bcc_self disabled
+        info!(
+            context,
+            "message {} has no recipient, skipping smtp-send", msg_id
+        );
+        set_delivered(context, msg_id).await?;
+        return Ok(None);
+    }
+
+    let rendered_msg = match mimefactory.render(context).await {
+        Ok(res) => Ok(res),
+        Err(err) => {
+            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+            Err(err)
+        }
+    }?;
+
+    if needs_encryption && !rendered_msg.is_encrypted {
+        /* unrecoverable */
+        message::set_msg_failed(
+            context,
+            msg_id,
+            Some("End-to-end-encryption unavailable unexpectedly."),
+        )
+        .await;
+        bail!(
+            "e2e encryption unavailable {} - {:?}",
+            msg_id,
+            needs_encryption
+        );
+    }
+
+    if rendered_msg.is_gossiped {
+        msg.chat_id.set_gossiped_timestamp(context, time()).await?;
+    }
+
+    if 0 != rendered_msg.last_added_location_id {
+        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, time()).await {
+            error!(context, "Failed to set kml sent_timestamp: {:?}", err);
+        }
+        if !msg.hidden {
+            if let Err(err) =
+                location::set_msg_location_id(context, msg.id, rendered_msg.last_added_location_id)
+                    .await
+            {
+                error!(context, "Failed to set msg_location_id: {:?}", err);
+            }
+        }
+    }
+
+    if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
+        if let Err(err) = context.delete_sync_ids(sync_ids).await {
+            error!(context, "Failed to delete sync ids: {:?}", err);
+        }
+    }
+
+    if attach_selfavatar {
+        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
+            error!(context, "Failed to set selfavatar timestamp: {:?}", err);
+        }
+    }
+
+    if rendered_msg.is_encrypted && !needs_encryption {
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
+        msg.update_param(context).await;
+    }
+
+    ensure!(!recipients.is_empty(), "no recipients for smtp job set");
+    let mut param = Params::new();
+    let bytes = &rendered_msg.message;
+    let blob = BlobObject::create(context, &rendered_msg.rfc724_mid, bytes).await?;
+
+    let recipients = recipients.join("\x1e");
+    param.set(Param::File, blob.as_name());
+    param.set(Param::Recipients, &recipients);
+
+    msg.subject = rendered_msg.subject.clone();
+    msg.update_subject(context).await;
+
+    let job = create(Action::SendMsgToSmtp, msg_id.to_u32(), param, 0)?;
+
+    Ok(Some(job))
+}
+
 pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
@@ -565,9 +999,20 @@ async fn perform_job_action(
     );
 
     let try_res = match job.action {
+        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
+        Action::SendMsgToSmtp => job.send_msg_to_smtp(context, connection.smtp()).await,
         Action::SendMdn => job.send_mdn(context, connection.smtp()).await,
+        Action::MaybeSendLocations => location::job_maybe_send_locations(context, job).await,
+        Action::MaybeSendLocationsEnded => {
+            location::job_maybe_send_locations_ended(context, job).await
+        }
         Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
+        Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
+        Action::Housekeeping => {
+            sql::housekeeping(context).await.ok_or_log(context);
+            Status::Finished(Ok(()))
+        }
         Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
             Ok(status) => status,
             Err(err) => Status::Finished(Err(err)),
@@ -595,20 +1040,16 @@ fn get_backoff_time_offset(tries: u32, action: Action) -> i64 {
             if seconds < 1 {
                 seconds = 1;
             }
-            i64::from(seconds)
+            seconds as i64
         }
     }
 }
 
-pub(crate) async fn send_mdn(context: &Context, msg_id: MsgId, from_id: ContactId) -> Result<()> {
+async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     let mut param = Params::new();
-    param.set(Param::MsgId, msg_id.to_u32().to_string());
+    param.set(Param::MsgId, msg.id.to_u32().to_string());
 
-    add(
-        context,
-        Job::new(Action::SendMdn, from_id.to_u32(), param, 0),
-    )
-    .await?;
+    add(context, Job::new(Action::SendMdn, msg.from_id, param, 0)).await?;
 
     Ok(())
 }
@@ -623,6 +1064,16 @@ pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
     Ok(())
 }
 
+/// Creates a job.
+pub fn create(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Result<Job> {
+    ensure!(
+        action != Action::Unknown,
+        "Invalid action passed to job_add"
+    );
+
+    Ok(Job::new(action, foreign_id, param, delay_seconds))
+}
+
 /// Adds a job to the database, scheduling it.
 pub async fn add(context: &Context, job: Job) -> Result<()> {
     let action = job.action;
@@ -631,14 +1082,20 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
 
     if delay_seconds == 0 {
         match action {
-            Action::ResyncFolders
+            Action::Unknown => unreachable!(),
+            Action::Housekeeping
+            | Action::ResyncFolders
+            | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
             | Action::UpdateRecentQuota
             | Action::DownloadMsg => {
                 info!(context, "interrupt: imap");
                 context.interrupt_inbox(InterruptInfo::new(false)).await;
             }
-            Action::SendMdn => {
+            Action::MaybeSendLocations
+            | Action::MaybeSendLocationsEnded
+            | Action::SendMdn
+            | Action::SendMsgToSmtp => {
                 info!(context, "interrupt: smtp");
                 context.interrupt_smtp(InterruptInfo::new(false)).await;
             }
@@ -647,6 +1104,18 @@ pub async fn add(context: &Context, job: Job) -> Result<()> {
     Ok(())
 }
 
+async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
+    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
+
+    let next_time = last_time + (60 * 60 * 24);
+    if next_time <= time() {
+        kill_action(context, Action::Housekeeping).await?;
+        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
+    } else {
+        Ok(None)
+    }
+}
+
 /// Load jobs from the database.
 ///
 /// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
@@ -690,7 +1159,7 @@ LIMIT 1;
         params = paramsv![thread_i];
     };
 
-    loop {
+    let job = loop {
         let job_res = context
             .sql
             .query_row_optional(query, params.clone(), |row| {
@@ -709,7 +1178,7 @@ LIMIT 1;
             .await;
 
         match job_res {
-            Ok(job) => return Ok(job),
+            Ok(job) => break job,
             Err(err) => {
                 // Remove invalid job from the DB
                 info!(context, "cleaning up job, because of {}", err);
@@ -727,6 +1196,20 @@ LIMIT 1;
                     .with_context(|| format!("Failed to delete invalid job {}", id))?;
             }
         }
+    };
+
+    match thread {
+        Thread::Unknown => {
+            bail!("unknown thread for job")
+        }
+        Thread::Imap => {
+            if let Some(job) = job {
+                Ok(Some(job))
+            } else {
+                Ok(load_housekeeping_job(context).await?)
+            }
+        }
+        Thread::Smtp => Ok(job),
     }
 }
 
@@ -774,7 +1257,8 @@ mod tests {
             &InterruptInfo::new(false),
         )
         .await?;
-        assert!(jobs.is_none());
+        // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
+        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -4,7 +4,7 @@ use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
-use anyhow::{format_err, Context as _, Result};
+use anyhow::{format_err, Result};
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
@@ -50,7 +50,8 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes)).context("rPGP error")
+        Self::KeyType::from_armor_single(Cursor::new(bytes))
+            .map_err(|err| format_err!("rPGP error: {}", err))
     }
 
     /// Load the users' default key from the database.
@@ -91,17 +92,16 @@ impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
 
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
-        let addr = context.get_primary_self_addr().await?;
         match context
             .sql
             .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
-             WHERE addr=?
+             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-                paramsv![addr],
+                paramsv![],
                 |row| {
                     let bytes: Vec<u8> = row.get(0)?;
                     Ok(bytes)
@@ -199,7 +199,10 @@ impl DcSecretKey for SignedSecretKey {
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context.get_primary_self_addr().await?;
+    let addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .ok_or_else(|| format_err!("No address configured"))?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -286,17 +289,17 @@ pub async fn store_self_keypair(
             paramsv![public_key, secret_key],
         )
         .await
-        .context("failed to remove old use of key")?;
+        .map_err(|err| err.context("failed to remove old use of key"))?;
     if default == KeyPairUse::Default {
         context
             .sql
             .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
-            .context("failed to clear default")?;
+            .map_err(|err| err.context("failed to clear default"))?;
     }
     let is_default = match default {
-        KeyPairUse::Default => i32::from(true),
-        KeyPairUse::ReadOnly => i32::from(false),
+        KeyPairUse::Default => true as i32,
+        KeyPairUse::ReadOnly => false as i32,
     };
 
     let addr = keypair.addr.to_string();
@@ -310,13 +313,13 @@ pub async fn store_self_keypair(
             paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
-        .context("failed to insert keypair")?;
+        .map_err(|err| err.context("failed to insert keypair"))?;
 
     Ok(())
 }
 
 /// A key fingerprint
-#[derive(Clone, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {

src/lib.rs

@@ -7,8 +7,7 @@
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
-    clippy::needless_borrow,
-    clippy::cast_lossless
+    clippy::needless_borrow
 )]
 #![allow(
     clippy::match_bool,
@@ -88,7 +87,6 @@ pub mod stock_str;
 mod sync;
 mod token;
 mod update_helper;
-pub mod webxdc;
 #[macro_use]
 mod dehtml;
 mod color;

src/location.rs

@@ -1,20 +1,20 @@
 //! Location handling.
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Context as _, Result};
-use async_std::channel::Receiver;
-use async_std::future::timeout;
+use anyhow::{ensure, Result};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
-use std::time::Duration;
 
 use crate::chat::{self, ChatId};
-use crate::contact::ContactId;
+use crate::config::Config;
+use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
-use crate::dc_tools::{duration_to_str, time};
+use crate::dc_tools::time;
 use crate::events::EventType;
-use crate::message::{Message, MsgId, Viewtype};
+use crate::job::{self, Job};
+use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
+use crate::param::Params;
 use crate::stock_str;
 
 /// Location record
@@ -25,7 +25,7 @@ pub struct Location {
     pub longitude: f64,
     pub accuracy: f64,
     pub timestamp: i64,
-    pub contact_id: ContactId,
+    pub contact_id: u32,
     pub msg_id: u32,
     pub chat_id: ChatId,
     pub marker: Option<String>,
@@ -101,10 +101,10 @@ impl Kml {
             let val = event.unescape_and_decode(reader).unwrap_or_default();
 
             let val = val
-                .replace('\n', "")
-                .replace('\r', "")
-                .replace('\t', "")
-                .replace(' ', "");
+                .replace("\n", "")
+                .replace("\r", "")
+                .replace("\t", "")
+                .replace(" ", "");
 
             if self.tag.contains(KmlTag::WHEN) && val.len() >= 19 {
                 // YYYY-MM-DDTHH:MM:SSZ
@@ -227,11 +227,32 @@ pub async fn send_locations_to_chat(
     }
     context.emit_event(EventType::ChatModified(chat_id));
     if 0 != seconds {
-        context.interrupt_location().await;
+        schedule_maybe_send_locations(context, false).await?;
+        job::add(
+            context,
+            job::Job::new(
+                job::Action::MaybeSendLocationsEnded,
+                chat_id.to_u32(),
+                Params::new(),
+                seconds + 1,
+            ),
+        )
+        .await?;
     }
     Ok(())
 }
 
+async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) -> Result<()> {
+    if force_schedule || !job::action_exists(context, job::Action::MaybeSendLocations).await? {
+        job::add(
+            context,
+            job::Job::new(job::Action::MaybeSendLocations, 0, Params::new(), 60),
+        )
+        .await?;
+    };
+    Ok(())
+}
+
 /// Returns whether `chat_id` or any chat is sending locations.
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
@@ -293,18 +314,18 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                         accuracy,
                         time(),
                         chat_id,
-                        ContactId::SELF,
+                        DC_CONTACT_ID_SELF,
                     ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
-                info!(context, "stored location for chat {}", chat_id);
                 continue_streaming = true;
             }
         }
         if continue_streaming {
-            context.emit_event(EventType::LocationChanged(Some(ContactId::SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
+        schedule_maybe_send_locations(context, false).await.ok();
     }
 
     continue_streaming
@@ -403,7 +424,10 @@ pub async fn delete_all(context: &Context) -> Result<()> {
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
     let mut last_added_location_id = 0;
 
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = context
+        .get_config(Config::ConfiguredAddr)
+        .await?
+        .unwrap_or_default();
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
@@ -438,10 +462,10 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
              GROUP BY timestamp \
              ORDER BY timestamp;",
                 paramsv![
-                    ContactId::SELF,
+                    DC_CONTACT_ID_SELF,
                     locations_send_begin,
                     locations_last_sent,
-                    ContactId::SELF
+                    DC_CONTACT_ID_SELF
                 ],
                 |row| {
                     let location_id: i32 = row.get(0)?;
@@ -534,7 +558,7 @@ pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id:
 pub(crate) async fn save(
     context: &Context,
     chat_id: ChatId,
-    contact_id: ContactId,
+    contact_id: u32,
     locations: &[Location],
     independent: bool,
 ) -> Result<Option<u32>> {
@@ -561,12 +585,12 @@ pub(crate) async fn save(
             conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
         let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
 
-        let exists = stmt_test.exists(paramsv![timestamp, contact_id])?;
+        let exists = stmt_test.exists(paramsv![timestamp, contact_id as i32])?;
 
         if independent || !exists {
             stmt_insert.execute(paramsv![
                 timestamp,
-                contact_id,
+                contact_id as i32,
                 chat_id,
                 latitude,
                 longitude,
@@ -587,140 +611,147 @@ pub(crate) async fn save(
     Ok(newest_location_id)
 }
 
-pub(crate) async fn location_loop(context: &Context, interrupt_receiver: Receiver<()>) {
-    loop {
-        let next_event = match maybe_send_locations(context).await {
-            Err(err) => {
-                warn!(context, "maybe_send_locations failed: {}", err);
-                Some(60) // Retry one minute later.
-            }
-            Ok(next_event) => next_event,
-        };
-
-        let duration = if let Some(next_event) = next_event {
-            Duration::from_secs(next_event)
-        } else {
-            Duration::from_secs(86400)
-        };
-
-        info!(
-            context,
-            "Location loop is waiting for {} or interrupt",
-            duration_to_str(duration)
-        );
-        timeout(duration, interrupt_receiver.recv()).await.ok();
-    }
-}
-
-/// Returns number of seconds until the next time location streaming for some chat ends
-/// automatically.
-async fn maybe_send_locations(context: &Context) -> Result<Option<u64>> {
-    let mut next_event: Option<u64> = None;
-
+pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
+    let mut continue_streaming = false;
+    info!(
+        context,
+        " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
+    );
+
     let rows = context
         .sql
         .query_map(
-            "SELECT id, locations_send_begin, locations_send_until, locations_last_sent
-             FROM chats
-             WHERE locations_send_until>0",
-            [],
+            "SELECT id, locations_send_begin, locations_last_sent \
+         FROM chats \
+         WHERE locations_send_until>?;",
+            paramsv![now],
             |row| {
                 let chat_id: ChatId = row.get(0)?;
                 let locations_send_begin: i64 = row.get(1)?;
-                let locations_send_until: i64 = row.get(2)?;
-                let locations_last_sent: i64 = row.get(3)?;
-                Ok((
-                    chat_id,
-                    locations_send_begin,
-                    locations_send_until,
-                    locations_last_sent,
-                ))
+                let locations_last_sent: i64 = row.get(2)?;
+                continue_streaming = true;
+
+                // be a bit tolerant as the timer may not align exactly with time(NULL)
+                if now - locations_last_sent < (60 - 3) {
+                    Ok(None)
+                } else {
+                    Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
+                }
             },
             |rows| {
-                rows.collect::<std::result::Result<Vec<_>, _>>()
+                rows.filter_map(|v| v.transpose())
+                    .collect::<std::result::Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
-        .await
-        .context("failed to query location streaming chats")?;
+        .await;
 
-    for (chat_id, locations_send_begin, locations_send_until, locations_last_sent) in rows {
-        if locations_send_begin > 0 && locations_send_until > now {
-            let can_send = now > locations_last_sent + 60;
-            let has_locations = context
-                .sql
-                .exists(
-                    "SELECT COUNT(id) \
+    if let Ok(rows) = rows {
+        let mut msgs = Vec::new();
+
+        {
+            let conn = job_try!(context.sql.get_conn().await);
+
+            let mut stmt_locations = job_try!(conn.prepare_cached(
+                "SELECT id \
      FROM locations \
      WHERE from_id=? \
      AND timestamp>=? \
      AND timestamp>? \
-     AND independent=0",
-                    paramsv![ContactId::SELF, locations_send_begin, locations_last_sent,],
-                )
-                .await?;
+     AND independent=0 \
+             ORDER BY timestamp;",
+            ));
 
-            next_event = next_event
-                .into_iter()
-                .chain(u64::try_from(locations_send_until - now).into_iter())
-                .min();
-
-            if has_locations {
-                if can_send {
-                    // Send location-only message.
-                    // Pending locations are attached automatically to every message,
+            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
+                if !stmt_locations
+                    .exists(paramsv![
+                        DC_CONTACT_ID_SELF,
+                        *locations_send_begin,
+                        *locations_last_sent,
+                    ])
+                    .unwrap_or_default()
+                {
+                    // if there is no new location, there's nothing to send.
+                    // however, maybe we want to bypass this test eg. 15 minutes
+                } else {
+                    // pending locations are attached automatically to every message,
                     // so also to this empty text message.
-                    info!(
-                        context,
-                        "Chat {} has pending locations, sending them.", chat_id
-                    );
+                    // DC_CMD_LOCATION is only needed to create a nicer subject.
+                    //
+                    // for optimisation and to avoid flooding the sending queue,
+                    // we could sending these messages only if we're really online.
+                    // the easiest way to determine this, is to check for an empty message queue.
+                    // (might not be 100%, however, as positions are sent combined later
+                    // and dc_set_location() is typically called periodically, this is ok)
                     let mut msg = Message::new(Viewtype::Text);
                     msg.hidden = true;
                     msg.param.set_cmd(SystemMessage::LocationOnly);
-                    chat::send_msg(context, chat_id, &mut msg).await?;
-                } else {
-                    // Wait until pending locations can be sent.
-                    info!(
-                        context,
-                        "Chat {} has pending locations, but they can't be sent yet.", chat_id
-                    );
-                    next_event = next_event
-                        .into_iter()
-                        .chain(u64::try_from(locations_last_sent + 61 - now).into_iter())
-                        .min();
+                    msgs.push((*chat_id, msg));
                 }
-            } else {
-                info!(
-                    context,
-                    "Chat {} has location streaming enabled, but no pending locations.", chat_id
-                );
             }
-        } else {
-            // Location streaming was either explicitly disabled (locations_send_begin = 0) or
-            // locations_send_until is in the past.
-            info!(
-                context,
-                "Disabling location streaming for chat {}.", chat_id
-            );
-            context
-                .sql
-                .execute(
-                    "UPDATE chats \
-                         SET locations_send_begin=0, locations_send_until=0 \
-                         WHERE id=?",
-                    paramsv![chat_id],
-                )
-                .await
-                .context("failed to disable location streaming")?;
+        }
 
-            let stock_str = stock_str::msg_location_disabled(context).await;
-            chat::add_info_msg(context, chat_id, &stock_str, now).await?;
-            context.emit_event(EventType::ChatModified(chat_id));
+        for (chat_id, mut msg) in msgs.into_iter() {
+            // TODO: better error handling
+            chat::send_msg(context, chat_id, &mut msg)
+                .await
+                .unwrap_or_default();
         }
     }
 
-    Ok(next_event)
+    if continue_streaming {
+        job_try!(schedule_maybe_send_locations(context, true).await);
+    }
+    job::Status::Finished(Ok(()))
+}
+
+pub(crate) async fn job_maybe_send_locations_ended(
+    context: &Context,
+    job: &mut Job,
+) -> job::Status {
+    // this function is called when location-streaming _might_ have ended for a chat.
+    // the function checks, if location-streaming is really ended;
+    // if so, a device-message is added if not yet done.
+
+    let chat_id = ChatId::new(job.foreign_id);
+
+    let (send_begin, send_until) = job_try!(
+        context
+            .sql
+            .query_row(
+                "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
+                paramsv![chat_id],
+                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
+            )
+            .await
+    );
+
+    let now = time();
+    if !(send_begin != 0 && now <= send_until) {
+        // still streaming -
+        // may happen as several calls to dc_send_locations_to_chat()
+        // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
+        if !(send_begin == 0 && send_until == 0) {
+            // not streaming, device-message already sent
+            job_try!(
+                context
+                    .sql
+                    .execute(
+                        "UPDATE chats \
+                             SET locations_send_begin=0, locations_send_until=0 \
+ WHERE id=?",
+                        paramsv![chat_id],
+                    )
+                    .await
+            );
+
+            let stock_str = stock_str::msg_location_disabled(context).await;
+            job_try!(chat::add_info_msg(context, chat_id, &stock_str, now).await);
+            context.emit_event(EventType::ChatModified(chat_id));
+        }
+    }
+    job::Status::Finished(Ok(()))
 }
 
 #[cfg(test)]
@@ -728,7 +759,6 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils::TestContext;
 
     #[async_std::test]
@@ -789,68 +819,4 @@ mod tests {
         assert!(!is_marker(" "));
         assert!(!is_marker("\t"));
     }
-
-    /// Tests that location.kml is hidden.
-    #[async_std::test]
-    async fn receive_location_kml() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        dc_receive_imf(
-            &alice,
-            br#"Subject: Hello
-Message-ID: hello@example.net
-To: Alice <alice@example.org>
-From: Bob <bob@example.net>
-Date: Mon, 20 Dec 2021 00:00:00 +0000
-Chat-Version: 1.0
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Text message."#,
-            false,
-        )
-        .await?;
-        let received_msg = alice.get_last_msg().await;
-        assert_eq!(received_msg.text.unwrap(), "Text message.");
-
-        dc_receive_imf(
-            &alice,
-            br#"Subject: locations
-MIME-Version: 1.0
-To: <alice@example.org>
-From: <bob@example.net>
-Date: Tue, 21 Dec 2021 00:00:00 +0000
-Chat-Version: 1.0
-Message-ID: <foobar@example.net>
-Content-Type: multipart/mixed; boundary="U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF"
-
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF
-Content-Type: application/vnd.google-earth.kml+xml
-Content-Disposition: attachment; filename="location.kml"
-
-<?xml version="1.0" encoding="UTF-8"?>
-<kml xmlns="http://www.opengis.net/kml/2.2">
-<Document addr="bob@example.net">
-<Placemark><Timestamp><when>2021-11-21T00:00:00Z</when></Timestamp><Point><coordinates accuracy="1.0000000000000000">10.00000000000000,20.00000000000000</coordinates></Point></Placemark>
-</Document>
-</kml>
-
---U8BOG8qNXfB0GgLiQ3PKUjlvdIuLRF--"#,
-            false,
-        )
-        .await?;
-
-        // Received location message is not visible, last message stays the same.
-        let received_msg2 = alice.get_last_msg().await;
-        assert_eq!(received_msg2.id, received_msg.id);
-
-        let locations = get_range(&alice, None, None, 0, 0).await?;
-        assert_eq!(locations.len(), 1);
-        Ok(())
-    }
 }

src/login_param.rs

@@ -139,18 +139,8 @@ pub struct LoginParam {
 }
 
 impl LoginParam {
-    /// Load entered (candidate) account settings
-    pub async fn load_candidate_params(context: &Context) -> Result<Self> {
-        LoginParam::from_database(context, "").await
-    }
-
-    /// Load configured (working) account settings
-    pub async fn load_configured_params(context: &Context) -> Result<Self> {
-        LoginParam::from_database(context, "configured_").await
-    }
-
     /// Read the login parameters from the database.
-    async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -252,18 +242,18 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_as_configured_params(&self, context: &Context) -> Result<()> {
-        let prefix = "configured_";
+    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
+        let prefix = prefix.as_ref();
         let sql = &context.sql;
 
-        context.set_primary_self_addr(&self.addr).await?;
+        let key = format!("{}addr", prefix);
+        sql.set_raw_config(key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
         sql.set_raw_config(key, Some(&self.imap.server)).await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(key, i32::from(self.imap.port))
-            .await?;
+        sql.set_raw_config_int(key, self.imap.port as i32).await?;
 
         let key = format!("{}mail_user", prefix);
         sql.set_raw_config(key, Some(&self.imap.user)).await?;
@@ -283,8 +273,7 @@ impl LoginParam {
         sql.set_raw_config(key, Some(&self.smtp.server)).await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(key, i32::from(self.smtp.port))
-            .await?;
+        sql.set_raw_config_int(key, self.smtp.port as i32).await?;
 
         let key = format!("{}send_user", prefix);
         sql.set_raw_config(key, Some(&self.smtp.user)).await?;
@@ -447,8 +436,8 @@ mod tests {
             socks5_config: None,
         };
 
-        param.save_as_configured_params(&t).await?;
-        let loaded = LoginParam::load_configured_params(&t).await?;
+        param.save_to_database(&t, "foobar_").await?;
+        let loaded = LoginParam::from_database(&t, "foobar_").await?;
 
         assert_eq!(param, loaded);
         Ok(())

src/message.rs

@@ -1,6 +1,7 @@
 //! # Messages and their identifiers.
 
-use std::collections::BTreeSet;
+use std::collections::BTreeMap;
+use std::convert::TryInto;
 
 use anyhow::{ensure, format_err, Context as _, Result};
 use async_std::path::{Path, PathBuf};
@@ -9,27 +10,25 @@ use rusqlite::types::ValueRef;
 use serde::{Deserialize, Serialize};
 
 use crate::chat::{self, Chat, ChatId};
-use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, VideochatType, DC_CHAT_ID_TRASH, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
+    Blocked, Chattype, VideochatType, Viewtype, DC_CHAT_ID_TRASH, DC_CONTACT_ID_INFO,
+    DC_CONTACT_ID_SELF, DC_DESIRED_TEXT_LEN, DC_MSG_ID_LAST_SPECIAL,
 };
-use crate::contact::{Contact, ContactId, Origin};
+use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::dc_tools::{
     dc_create_smeared_timestamp, dc_get_filebytes, dc_get_filemeta, dc_gm2local_offset,
     dc_read_file, dc_timestamp_to_str, dc_truncate, time,
 };
 use crate::download::DownloadState;
-use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::events::EventType;
-use crate::imap::markseen_on_imap_table;
-use crate::job;
+use crate::job::{self, Action};
 use crate::log::LogExt;
 use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::scheduler::InterruptInfo;
-use crate::sql;
 use crate::stock_str;
 use crate::summary::Summary;
 
@@ -114,25 +113,14 @@ WHERE id=?;
         Ok(())
     }
 
-    /// Deletes a message, corresponding MDNs and unsent SMTP messages from the database.
+    /// Deletes a message and corresponding MDNs from the database.
     pub async fn delete_from_db(self, context: &Context) -> Result<()> {
         // We don't use transactions yet, so remove MDNs first to make
         // sure they are not left while the message is deleted.
-        context
-            .sql
-            .execute("DELETE FROM smtp WHERE msg_id=?", paramsv![self])
-            .await?;
         context
             .sql
             .execute("DELETE FROM msgs_mdns WHERE msg_id=?;", paramsv![self])
             .await?;
-        context
-            .sql
-            .execute(
-                "DELETE FROM msgs_status_updates WHERE msg_id=?;",
-                paramsv![self],
-            )
-            .await?;
         context
             .sql
             .execute("DELETE FROM msgs WHERE id=?;", paramsv![self])
@@ -140,20 +128,6 @@ WHERE id=?;
         Ok(())
     }
 
-    pub(crate) async fn set_delivered(self, context: &Context) -> Result<()> {
-        update_msg_state(context, self, MessageState::OutDelivered).await?;
-        let chat_id: ChatId = context
-            .sql
-            .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![self])
-            .await?
-            .unwrap_or_default();
-        context.emit_event(EventType::MsgDelivered {
-            chat_id,
-            msg_id: self,
-        });
-        Ok(())
-    }
-
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -181,10 +155,10 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(
-                format_err!("Invalid MsgId {}", self.0).into(),
+                format_err!("Invalid MsgId").into(),
             ));
         }
-        let val = rusqlite::types::Value::Integer(i64::from(self.0));
+        let val = rusqlite::types::Value::Integer(self.0 as i64);
         let out = rusqlite::types::ToSqlOutput::Owned(val);
         Ok(out)
     }
@@ -195,7 +169,7 @@ impl rusqlite::types::FromSql for MsgId {
     fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
         // Would be nice if we could use match here, but alas.
         i64::column_result(value).and_then(|val| {
-            if 0 <= val && val <= i64::from(std::u32::MAX) {
+            if 0 <= val && val <= std::u32::MAX as i64 {
                 Ok(MsgId::new(val as u32))
             } else {
                 Err(rusqlite::types::FromSqlError::OutOfRange(val))
@@ -241,8 +215,8 @@ impl Default for MessengerMessage {
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct Message {
     pub(crate) id: MsgId,
-    pub(crate) from_id: ContactId,
-    pub(crate) to_id: ContactId,
+    pub(crate) from_id: u32,
+    pub(crate) to_id: u32,
     pub(crate) chat_id: ChatId,
     pub(crate) viewtype: Viewtype,
     pub(crate) state: MessageState,
@@ -388,7 +362,7 @@ impl Message {
     }
 
     pub async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> {
-        if self.viewtype.has_file() {
+        if chat::msgtype_has_file(self.viewtype) {
             let file_param = self.param.get_path(Param::File, context)?;
             if let Some(path_and_filename) = file_param {
                 if (self.viewtype == Viewtype::Image || self.viewtype == Viewtype::Gif)
@@ -432,7 +406,7 @@ impl Message {
     /// this is done by dc_set_location() and dc_send_locations_to_chat().
     ///
     /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
-    /// contact_id set to ContactId::SELF.
+    /// contact_id set to DC_CONTACT_ID_SELF.
     ///
     /// @param latitude North-south position of the location.
     /// @param longitude East-west position of the location.
@@ -457,7 +431,7 @@ impl Message {
         self.id
     }
 
-    pub fn get_from_id(&self) -> ContactId {
+    pub fn get_from_id(&self) -> u32 {
         self.from_id
     }
 
@@ -545,7 +519,7 @@ impl Message {
             &chat_loaded
         };
 
-        let contact = if self.from_id != ContactId::SELF {
+        let contact = if self.from_id != DC_CONTACT_ID_SELF {
             match chat.typ {
                 Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                     Some(Contact::get_by_id(context, self.from_id).await?)
@@ -598,8 +572,8 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == ContactId::INFO
-            || self.to_id == ContactId::INFO
+        self.from_id == DC_CONTACT_ID_INFO
+            || self.to_id == DC_CONTACT_ID_INFO
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
@@ -621,7 +595,7 @@ impl Message {
     /// copied to the blobdir.  Thus those attachments need to be
     /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
-        self.viewtype.has_file() && self.state == MessageState::OutPreparing
+        chat::msgtype_has_file(self.viewtype) && self.state == MessageState::OutPreparing
     }
 
     pub fn is_setupmessage(&self) -> bool {
@@ -782,41 +756,36 @@ impl Message {
     ///
     /// The message itself is not required to exist in the database,
     /// it may even be deleted from the database by the time the message is prepared.
-    pub async fn set_quote(&mut self, context: &Context, quote: Option<&Message>) -> Result<()> {
-        if let Some(quote) = quote {
-            ensure!(
-                !quote.rfc724_mid.is_empty(),
-                "Message without Message-Id cannot be quoted"
-            );
-            self.in_reply_to = Some(quote.rfc724_mid.clone());
+    pub async fn set_quote(&mut self, context: &Context, quote: &Message) -> Result<()> {
+        ensure!(
+            !quote.rfc724_mid.is_empty(),
+            "Message without Message-Id cannot be quoted"
+        );
+        self.in_reply_to = Some(quote.rfc724_mid.clone());
 
-            if quote
-                .param
-                .get_bool(Param::GuaranteeE2ee)
-                .unwrap_or_default()
-            {
-                self.param.set(Param::GuaranteeE2ee, "1");
-            }
-
-            let text = quote.get_text().unwrap_or_default();
-            self.param.set(
-                Param::Quote,
-                if text.is_empty() {
-                    // Use summary, similar to "Image" to avoid sending empty quote.
-                    quote
-                        .get_summary(context, None)
-                        .await?
-                        .truncated_text(500)
-                        .to_string()
-                } else {
-                    text
-                },
-            );
-        } else {
-            self.in_reply_to = None;
-            self.param.remove(Param::Quote);
+        if quote
+            .param
+            .get_bool(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+        {
+            self.param.set(Param::GuaranteeE2ee, "1");
         }
 
+        let text = quote.get_text().unwrap_or_default();
+        self.param.set(
+            Param::Quote,
+            if text.is_empty() {
+                // Use summary, similar to "Image" to avoid sending empty quote.
+                quote
+                    .get_summary(context, None)
+                    .await?
+                    .truncated_text(500)
+                    .to_string()
+            } else {
+                text
+            },
+        );
+
         Ok(())
     }
 
@@ -826,21 +795,16 @@ impl Message {
 
     pub async fn quoted_message(&self, context: &Context) -> Result<Option<Message>> {
         if self.param.get(Param::Quote).is_some() && !self.is_forwarded() {
-            return self.parent(context).await;
-        }
-        Ok(None)
-    }
-
-    pub async fn parent(&self, context: &Context) -> Result<Option<Message>> {
-        if let Some(in_reply_to) = &self.in_reply_to {
-            if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
-                let msg = Message::load_from_db(context, msg_id).await?;
-                return if msg.chat_id.is_trash() {
-                    // If message is already moved to trash chat, pretend it does not exist.
-                    Ok(None)
-                } else {
-                    Ok(Some(msg))
-                };
+            if let Some(in_reply_to) = &self.in_reply_to {
+                if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
+                    let msg = Message::load_from_db(context, msg_id).await?;
+                    return if msg.chat_id.is_trash() {
+                        // If message is already moved to trash chat, pretend it does not exist.
+                        Ok(None)
+                    } else {
+                        Ok(Some(msg))
+                    };
+                }
             }
         }
         Ok(None)
@@ -1018,7 +982,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
     ret += &format!(" by {}", name);
     ret += "\n";
 
-    if msg.from_id != ContactId::SELF {
+    if msg.from_id != DC_CONTACT_ID_SELF {
         let s = dc_timestamp_to_str(if 0 != msg.timestamp_rcvd {
             msg.timestamp_rcvd
         } else {
@@ -1039,7 +1003,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
         );
     }
 
-    if msg.from_id == ContactId::INFO || msg.to_id == ContactId::INFO {
+    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return Ok(ret);
     }
@@ -1050,7 +1014,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
             "SELECT contact_id, timestamp_sent FROM msgs_mdns WHERE msg_id=?;",
             paramsv![msg_id],
             |row| {
-                let contact_id: ContactId = row.get(0)?;
+                let contact_id: i32 = row.get(0)?;
                 let ts: i64 = row.get(1)?;
                 Ok((contact_id, ts))
             },
@@ -1062,7 +1026,7 @@ pub async fn get_msg_info(context: &Context, msg_id: MsgId) -> Result<String> {
             let fts = dc_timestamp_to_str(ts);
             ret += &format!("Read: {}", fts);
 
-            let name = Contact::load_from_db(context, contact_id)
+            let name = Contact::load_from_db(context, contact_id.try_into()?)
                 .await
                 .map(|contact| contact.get_name_n_addr())
                 .unwrap_or_default();
@@ -1192,7 +1156,6 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "webm" => (Viewtype::Video, "video/webm"),
         "webp" => (Viewtype::Image, "image/webp"), // iOS via SDWebImage, Android since 4.0
         "wmv" => (Viewtype::Video, "video/x-ms-wmv"),
-        "xdc" => (Viewtype::Webxdc, "application/webxdc+zip"),
         "xhtml" => (Viewtype::File, "application/xhtml+xml"),
         "xlsx" => (
             Viewtype::File,
@@ -1238,7 +1201,7 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     for msg_id in msg_ids.iter() {
         let msg = Message::load_from_db(context, *msg_id).await?;
         if msg.location_id > 0 {
-            delete_poi_location(context, msg.location_id).await?;
+            delete_poi_location(context, msg.location_id).await;
         }
         msg_id
             .trash(context)
@@ -1254,26 +1217,32 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     }
 
     if !msg_ids.is_empty() {
-        context.emit_msgs_changed_without_ids();
-
-        // Run housekeeping to delete unused blobs.
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
+        job::kill_action(context, Action::Housekeeping).await?;
+        job::add(
+            context,
+            job::Job::new(Action::Housekeeping, 0, Params::new(), 10),
+        )
+        .await?;
     }
 
-    // Interrupt Inbox loop to start message deletion and run housekeeping.
+    // Interrupt Inbox loop to start message deletion.
     context.interrupt_inbox(InterruptInfo::new(false)).await;
     Ok(())
 }
 
-async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
+async fn delete_poi_location(context: &Context, location_id: u32) -> bool {
     context
         .sql
         .execute(
             "DELETE FROM locations WHERE independent = 1 AND id=?;",
             paramsv![location_id as i32],
         )
-        .await?;
-    Ok(())
+        .await
+        .is_ok()
 }
 
 pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()> {
@@ -1281,118 +1250,81 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
         return Ok(());
     }
 
-    let msgs = context
-        .sql
-        .query_map(
-            format!(
-                "SELECT
-                    m.id AS id,
-                    m.chat_id AS chat_id,
-                    m.state AS state,
-                    m.ephemeral_timer AS ephemeral_timer,
-                    m.param AS param,
-                    m.from_id AS from_id,
-                    m.rfc724_mid AS rfc724_mid,
-                    c.blocked AS blocked
-                 FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id
-                 WHERE m.id IN ({}) AND m.chat_id>9",
-                sql::repeat_vars(msg_ids.len())
-            ),
-            rusqlite::params_from_iter(&msg_ids),
-            |row| {
-                let id: MsgId = row.get("id")?;
-                let chat_id: ChatId = row.get("chat_id")?;
-                let state: MessageState = row.get("state")?;
-                let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
-                let from_id: ContactId = row.get("from_id")?;
-                let rfc724_mid: String = row.get("rfc724_mid")?;
-                let blocked: Option<Blocked> = row.get("blocked")?;
-                let ephemeral_timer: EphemeralTimer = row.get("ephemeral_timer")?;
+    let conn = context.sql.get_conn().await?;
+    let msgs = async_std::task::spawn_blocking(move || -> Result<_> {
+        let mut stmt = conn.prepare_cached(concat!(
+            "SELECT",
+            "    m.chat_id AS chat_id,",
+            "    m.state AS state,",
+            "    c.blocked AS blocked",
+            " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+            " WHERE m.id=? AND m.chat_id>9"
+        ))?;
+
+        let mut msgs = Vec::with_capacity(msg_ids.len());
+        for id in msg_ids.into_iter() {
+            let query_res = stmt.query_row(paramsv![id], |row| {
                 Ok((
-                    id,
-                    chat_id,
-                    state,
-                    param,
-                    from_id,
-                    rfc724_mid,
-                    blocked.unwrap_or_default(),
-                    ephemeral_timer,
+                    row.get::<_, ChatId>("chat_id")?,
+                    row.get::<_, MessageState>("state")?,
+                    row.get::<_, Option<Blocked>>("blocked")?
+                        .unwrap_or_default(),
                 ))
-            },
-            |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
-        )
-        .await?;
+            });
+            if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
+                continue;
+            }
+            let (chat_id, state, blocked) = query_res.map_err(Into::<anyhow::Error>::into)?;
+            msgs.push((id, chat_id, state, blocked));
+        }
+        drop(stmt);
+        drop(conn);
+        Ok(msgs)
+    })
+    .await?;
 
-    if msgs.iter().any(
-        |(_id, _chat_id, _state, _param, _from_id, _rfc724_mid, _blocked, ephemeral_timer)| {
-            *ephemeral_timer != EphemeralTimer::Disabled
-        },
-    ) {
-        start_ephemeral_timers_msgids(context, &msg_ids)
-            .await
-            .context("failed to start ephemeral timers")?;
-    }
+    let mut updated_chat_ids = BTreeMap::new();
+
+    for (id, curr_chat_id, curr_state, curr_blocked) in msgs.into_iter() {
+        if let Err(err) = id.start_ephemeral_timer(context).await {
+            error!(
+                context,
+                "Failed to start ephemeral timer for message {}: {}", id, err
+            );
+            continue;
+        }
 
-    let mut updated_chat_ids = BTreeSet::new();
-    for (
-        id,
-        curr_chat_id,
-        curr_state,
-        curr_param,
-        curr_from_id,
-        curr_rfc724_mid,
-        curr_blocked,
-        _curr_ephemeral_timer,
-    ) in msgs.into_iter()
-    {
         if curr_blocked == Blocked::Not
             && (curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed)
         {
-            update_msg_state(context, id, MessageState::InSeen).await?;
+            update_msg_state(context, id, MessageState::InSeen).await;
             info!(context, "Seen message {}.", id);
 
-            markseen_on_imap_table(context, &curr_rfc724_mid).await?;
-
-            // Read receipts for system messages are never sent. These messages have no place to
-            // display received read receipt anyway.  And since their text is locally generated,
-            // quoting them is dangerous as it may contain contact names. E.g., for original message
-            // "Group left by me", a read receipt will quote "Group left by <name>", and the name can
-            // be a display name stored in address book rather than the name sent in the From field by
-            // the user.
-            if curr_param.get_bool(Param::WantsMdn).unwrap_or_default()
-                && curr_param.get_cmd() == SystemMessage::Unknown
-            {
-                let mdns_enabled = context.get_config_bool(Config::MdnsEnabled).await?;
-                if mdns_enabled {
-                    if let Err(err) = job::send_mdn(context, id, curr_from_id).await {
-                        warn!(context, "could not send out mdn for {}: {}", id, err);
-                    }
-                }
-            }
-            updated_chat_ids.insert(curr_chat_id);
+            job::add(
+                context,
+                job::Job::new(Action::MarkseenMsgOnImap, id.to_u32(), Params::new(), 0),
+            )
+            .await?;
+            updated_chat_ids.insert(curr_chat_id, true);
         }
     }
 
-    for updated_chat_id in updated_chat_ids {
-        context.emit_event(EventType::MsgsNoticed(updated_chat_id));
+    for updated_chat_id in updated_chat_ids.keys() {
+        context.emit_event(EventType::MsgsNoticed(*updated_chat_id));
     }
 
     Ok(())
 }
 
-pub(crate) async fn update_msg_state(
-    context: &Context,
-    msg_id: MsgId,
-    state: MessageState,
-) -> Result<()> {
+pub async fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageState) -> bool {
     context
         .sql
         .execute(
             "UPDATE msgs SET state=? WHERE id=?;",
             paramsv![state, msg_id],
         )
-        .await?;
-    Ok(())
+        .await
+        .is_ok()
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1454,11 +1386,11 @@ pub async fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl
 /// returns Some if an event should be send
 pub async fn handle_mdn(
     context: &Context,
-    from_id: ContactId,
+    from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Result<Option<(ChatId, MsgId)>> {
-    if from_id == ContactId::SELF {
+    if from_id == DC_CONTACT_ID_SELF {
         warn!(
             context,
             "ignoring MDN sent to self, this is a bug on the sender device"
@@ -1507,7 +1439,7 @@ pub async fn handle_mdn(
         .sql
         .exists(
             "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-            paramsv![msg_id, from_id],
+            paramsv![msg_id, from_id as i32,],
         )
         .await?
     {
@@ -1515,7 +1447,7 @@ pub async fn handle_mdn(
             .sql
             .execute(
                 "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                paramsv![msg_id, from_id, timestamp_sent],
+                paramsv![msg_id, from_id as i32, timestamp_sent],
             )
             .await?;
     }
@@ -1524,7 +1456,7 @@ pub async fn handle_mdn(
         || msg_state == MessageState::OutPending
         || msg_state == MessageState::OutDelivered
     {
-        update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await?;
+        update_msg_state(context, msg_id, MessageState::OutMdnRcvd).await;
         Ok(Some((chat_id, msg_id)))
     } else {
         Ok(None)
@@ -1593,7 +1525,9 @@ async fn ndn_maybe_add_info_msg(
                 let contact_id =
                     Contact::lookup_id_by_addr(context, failed_recipient, Origin::Unknown)
                         .await?
-                        .context("contact ID not found")?;
+                        .ok_or_else(|| {
+                            format_err!("ndn_maybe_add_info_msg: Contact ID not found")
+                        })?;
                 let contact = Contact::load_from_db(context, contact_id).await?;
                 // Tell the user which of the recipients failed if we know that (because in
                 // a group, this might otherwise be unclear)
@@ -1663,7 +1597,7 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+    let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
         .await?
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
@@ -1729,128 +1663,22 @@ pub(crate) async fn rfc724_mid_exists(
     Ok(res)
 }
 
-/// How a message is primarily displayed.
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
-#[repr(u32)]
-pub enum Viewtype {
-    Unknown = 0,
-
-    /// Text message.
-    /// The text of the message is set using dc_msg_set_text()
-    /// and retrieved with dc_msg_get_text().
-    Text = 10,
-
-    /// Image message.
-    /// If the image is an animated GIF, the type DC_MSG_GIF should be used.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension
-    /// and retrieved via dc_msg_set_file(), dc_msg_set_dimension().
-    Image = 20,
-
-    /// Animated GIF message.
-    /// File, width and height are set via dc_msg_set_file(), dc_msg_set_dimension()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
-    Gif = 21,
-
-    /// Message containing a sticker, similar to image.
-    /// If possible, the ui should display the image without borders in a transparent way.
-    /// A click on a sticker will offer to install the sticker set in some future.
-    Sticker = 23,
-
-    /// Message containing an Audio file.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
-    Audio = 40,
-
-    /// A voice message that was directly recorded by the user.
-    /// For all other audio messages, the type #DC_MSG_AUDIO should be used.
-    /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
-    /// and retrieved via dc_msg_get_file(), dc_msg_get_duration()
-    Voice = 41,
-
-    /// Video messages.
-    /// File, width, height and durarion
-    /// are set via dc_msg_set_file(), dc_msg_set_dimension(), dc_msg_set_duration()
-    /// and retrieved via
-    /// dc_msg_get_file(), dc_msg_get_width(),
-    /// dc_msg_get_height(), dc_msg_get_duration().
-    Video = 50,
-
-    /// Message containing any file, eg. a PDF.
-    /// The file is set via dc_msg_set_file()
-    /// and retrieved via dc_msg_get_file().
-    File = 60,
-
-    /// Message is an invitation to a videochat.
-    VideochatInvitation = 70,
-
-    /// Message is an webxdc instance.
-    Webxdc = 80,
-}
-
-impl Default for Viewtype {
-    fn default() -> Self {
-        Viewtype::Unknown
-    }
-}
-
-impl Viewtype {
-    /// Whether a message with this [`Viewtype`] should have a file attachment.
-    pub fn has_file(&self) -> bool {
-        match self {
-            Viewtype::Unknown => false,
-            Viewtype::Text => false,
-            Viewtype::Image => true,
-            Viewtype::Gif => true,
-            Viewtype::Sticker => true,
-            Viewtype::Audio => true,
-            Viewtype::Voice => true,
-            Viewtype::Video => true,
-            Viewtype::File => true,
-            Viewtype::VideochatInvitation => false,
-            Viewtype::Webxdc => true,
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
-    use num_traits::FromPrimitive;
-
+    use super::*;
     use crate::chat::{marknoticed_chat, ChatItem};
     use crate::chatlist::Chatlist;
+    use crate::constants::DC_CONTACT_ID_DEVICE;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::test_utils as test;
     use crate::test_utils::TestContext;
 
-    use super::*;
-
     #[test]
     fn test_guess_msgtype_from_suffix() {
         assert_eq!(
             guess_msgtype_from_suffix(Path::new("foo/bar-sth.mp3")),
             Some((Viewtype::Audio, "audio/mpeg"))
         );
-        assert_eq!(
-            guess_msgtype_from_suffix(Path::new("foo/file.html")),
-            Some((Viewtype::File, "text/html"))
-        );
-        assert_eq!(
-            guess_msgtype_from_suffix(Path::new("foo/file.xdc")),
-            Some((Viewtype::Webxdc, "application/webxdc+zip"))
-        );
     }
 
     #[async_std::test]
@@ -1970,7 +1798,7 @@ mod tests {
         // test that get_width() and get_height() are returning some dimensions for images;
         // (as the device-chat contains a welcome-images, we check that)
         t.update_device_chats().await.ok();
-        let device_chat_id = ChatId::get_for_contact(&t, ContactId::DEVICE)
+        let device_chat_id = ChatId::get_for_contact(&t, DC_CONTACT_ID_DEVICE)
             .await
             .unwrap();
 
@@ -2018,9 +1846,7 @@ mod tests {
         assert!(!msg.rfc724_mid.is_empty());
 
         let mut msg2 = Message::new(Viewtype::Text);
-        msg2.set_quote(ctx, Some(&msg))
-            .await
-            .expect("can't set quote");
+        msg2.set_quote(ctx, &msg).await.expect("can't set quote");
         assert!(msg2.quoted_text() == msg.get_text());
 
         let quoted_msg = msg2
@@ -2044,6 +1870,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await
@@ -2150,9 +1977,9 @@ mod tests {
         let msg2 = alice.get_last_msg().await;
         let chats = Chatlist::try_load(&alice, 0, None, None).await?;
         assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0)?, alice_chat.id);
-        assert_eq!(chats.get_chat_id(0)?, msg1.chat_id);
-        assert_eq!(chats.get_chat_id(0)?, msg2.chat_id);
+        assert_eq!(chats.get_chat_id(0), alice_chat.id);
+        assert_eq!(chats.get_chat_id(0), msg1.chat_id);
+        assert_eq!(chats.get_chat_id(0), msg2.chat_id);
         assert_eq!(alice_chat.id.get_fresh_msg_cnt(&alice).await?, 2);
         assert_eq!(alice.get_fresh_msgs().await?.len(), 2);
 
@@ -2216,7 +2043,7 @@ mod tests {
         let payload = alice.pop_sent_msg().await;
         assert_state(&alice, alice_msg.id, MessageState::OutDelivered).await;
 
-        update_msg_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await?;
+        update_msg_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await;
         assert_state(&alice, alice_msg.id, MessageState::OutMdnRcvd).await;
 
         set_msg_failed(&alice, alice_msg.id, Some("badly failed")).await;
@@ -2252,6 +2079,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -2269,6 +2097,7 @@ mod tests {
                     Date: Fri, 29 Jan 2021 21:37:55 +0000\n\
                     \n\
                     hello again\n",
+            "INBOX",
             false,
         )
         .await?;
@@ -2278,29 +2107,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[test]
-    fn test_viewtype_derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-
-    #[test]
-    fn test_viewtype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Viewtype::Unknown, Viewtype::default());
-        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
-        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
-        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
-        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
-        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
-        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
-        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
-        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
-        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
-        assert_eq!(
-            Viewtype::VideochatInvitation,
-            Viewtype::from_i32(70).unwrap()
-        );
-        assert_eq!(Viewtype::Webxdc, Viewtype::from_i32(80).unwrap());
-    }
 }

src/mimefactory.rs

@@ -2,14 +2,14 @@
 
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
 
 use crate::blob::BlobObject;
-use crate::chat::Chat;
+use crate::chat::{self, Chat};
 use crate::config::Config;
-use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
+use crate::constants::{Chattype, Viewtype, DC_FROM_HANDSHAKE};
 use crate::contact::Contact;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::IsNoneOrEmpty;
@@ -22,7 +22,7 @@ use crate::ephemeral::Timer as EphemeralTimer;
 use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::html::new_html_mimepart;
 use crate::location;
-use crate::message::{self, Message, MsgId, Viewtype};
+use crate::message::{self, Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
@@ -81,7 +81,7 @@ pub struct MimeFactory<'a> {
 /// Result of rendering a message, ready to be submitted to a send job.
 #[derive(Debug, Clone)]
 pub struct RenderedEmail {
-    pub message: String,
+    pub message: Vec<u8>,
     // pub envelope: Envelope,
     pub is_encrypted: bool,
     pub is_gossiped: bool,
@@ -133,7 +133,11 @@ impl<'a> MimeFactory<'a> {
     ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
-        let from_addr = context.get_primary_self_addr().await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
+
         let config_displayname = context
             .get_config(Config::Displayname)
             .await?
@@ -203,6 +207,7 @@ impl<'a> MimeFactory<'a> {
             )
             .await?;
 
+        let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
             from_addr,
             from_displayname,
@@ -210,7 +215,7 @@ impl<'a> MimeFactory<'a> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await?
-                .unwrap_or_default(),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -233,15 +238,19 @@ impl<'a> MimeFactory<'a> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
-        let from_addr = context.get_primary_self_addr().await?;
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .unwrap_or_default();
         let from_displayname = context
             .get_config(Config::Displayname)
             .await?
             .unwrap_or_default();
+        let default_str = stock_str::status_line(context).await;
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await?
-            .unwrap_or_default();
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a> {
@@ -271,7 +280,10 @@ impl<'a> MimeFactory<'a> {
         &self,
         context: &Context,
     ) -> Result<Vec<(Option<Peerstate>, &str)>> {
-        let self_addr = context.get_primary_self_addr().await?;
+        let self_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .await?
+            .ok_or_else(|| format_err!("Not configured"))?;
 
         let mut res = Vec::new();
         for (_, addr) in self
@@ -493,80 +505,33 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
-        // Start with Internet Message Format headers in the order of the standard example
-        // <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.1.1>.
         headers
             .unprotected
-            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
-        if let Some(sender_displayname) = &self.sender_displayname {
-            let sender =
-                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
-            headers
-                .unprotected
-                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
-        }
-        headers
-            .unprotected
-            .push(Header::new_with_value("To".into(), to).unwrap());
+            .push(Header::new("MIME-Version".into(), "1.0".into()));
 
-        let subject_str = self.subject_str(context).await?;
-        let encoded_subject = if subject_str
-            .chars()
-            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
-        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
-        // but we do not want to encode all subjects just because they contain a space.
-        {
-            subject_str.clone()
-        } else {
-            encode_words(&subject_str)
-        };
-        headers
-            .protected
-            .push(Header::new("Subject".into(), encoded_subject));
-
-        let date = chrono::Utc
-            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
-            .unwrap()
-            .to_rfc2822();
-        headers.unprotected.push(Header::new("Date".into(), date));
-
-        let rfc724_mid = match self.loaded {
-            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
-            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
-        };
-        let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
-
-        // Amazon's SMTP servers change the `Message-ID`, just as Outlook's SMTP servers do.
-        // Outlook's servers add an `X-Microsoft-Original-Message-ID` header with the original `Message-ID`,
-        // and when downloading messages we look for this header in order to correctly identify
-        // messages.
-        // Amazon's servers do not add such a header, so we just add it ourselves.
-        if let Some(server) = context.get_config(Config::ConfiguredSendServer).await? {
-            if server.ends_with(".amazonaws.com") {
-                headers.unprotected.push(Header::new(
-                    "X-Microsoft-Original-Message-ID".into(),
-                    rfc724_mid_headervalue.clone(),
-                ))
-            }
-        }
-
-        headers
-            .unprotected
-            .push(Header::new("Message-ID".into(), rfc724_mid_headervalue));
-
-        // Reply headers as in <https://datatracker.ietf.org/doc/html/rfc5322#appendix-A.2>.
-        if !self.in_reply_to.is_empty() {
-            headers
-                .unprotected
-                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
-        }
         if !self.references.is_empty() {
             headers
                 .unprotected
                 .push(Header::new("References".into(), self.references.clone()));
         }
 
-        // Automatic Response headers <https://www.rfc-editor.org/rfc/rfc3834>
+        if !self.in_reply_to.is_empty() {
+            headers
+                .unprotected
+                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
+        }
+
+        let date = chrono::Utc
+            .from_local_datetime(&chrono::NaiveDateTime::from_timestamp(self.timestamp, 0))
+            .unwrap()
+            .to_rfc2822();
+
+        headers.unprotected.push(Header::new("Date".into(), date));
+
+        headers
+            .unprotected
+            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
+
         if let Loaded::Mdn { .. } = self.loaded {
             headers.unprotected.push(Header::new(
                 "Auto-Submitted".to_string(),
@@ -579,11 +544,6 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
-        // Non-standard headers.
-        headers
-            .unprotected
-            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
-
         if self.req_mdn {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
@@ -598,9 +558,21 @@ impl<'a> MimeFactory<'a> {
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
         let skip_autocrypt = self.should_skip_autocrypt();
+        let subject_str = self.subject_str(context).await?;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let encrypt_helper = EncryptHelper::new(context).await?;
 
+        let encoded_subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str.clone()
+        } else {
+            encode_words(&subject_str)
+        };
+
         if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
@@ -609,6 +581,15 @@ impl<'a> MimeFactory<'a> {
                 .push(Header::new("Autocrypt".into(), aheader));
         }
 
+        headers
+            .protected
+            .push(Header::new("Subject".into(), encoded_subject));
+
+        let rfc724_mid = match self.loaded {
+            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
+            Loaded::Mdn { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
+        };
+
         let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(context).await?;
         if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
             headers.protected.push(Header::new(
@@ -617,11 +598,25 @@ impl<'a> MimeFactory<'a> {
             ));
         }
 
-        // MIME header <https://datatracker.ietf.org/doc/html/rfc2045>.
-        // Content-Type
+        headers.unprotected.push(Header::new(
+            "Message-ID".into(),
+            render_rfc724_mid(&rfc724_mid),
+        ));
+
         headers
             .unprotected
-            .push(Header::new("MIME-Version".into(), "1.0".into()));
+            .push(Header::new_with_value("To".into(), to).unwrap());
+
+        headers
+            .unprotected
+            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
+        if let Some(sender_displayname) = &self.sender_displayname {
+            let sender =
+                Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
+            headers
+                .unprotected
+                .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
+        }
 
         let mut is_gossiped = false;
 
@@ -648,11 +643,6 @@ impl<'a> MimeFactory<'a> {
                     "Content-Type".to_string(),
                     "multipart/report; report-type=multi-device-sync".to_string(),
                 ))
-            } else if self.msg.param.get_cmd() == SystemMessage::WebxdcStatusUpdate {
-                PartBuilder::new().header((
-                    "Content-Type".to_string(),
-                    "multipart/report; report-type=status-update".to_string(),
-                ))
             } else {
                 PartBuilder::new().message_type(MimeMultipartType::Mixed)
             };
@@ -777,7 +767,7 @@ impl<'a> MimeFactory<'a> {
         } = self;
 
         Ok(RenderedEmail {
-            message: outer_message.build().as_string(),
+            message: outer_message.build().as_string().into_bytes(),
             // envelope: Envelope::new,
             is_encrypted,
             is_gossiped,
@@ -850,12 +840,9 @@ impl<'a> MimeFactory<'a> {
         }
 
         if chat.typ == Chattype::Group {
-            // Send group ID unless it is an ad hoc group that has no ID.
-            if !chat.grpid.is_empty() {
-                headers
-                    .protected
-                    .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
-            }
+            headers
+                .protected
+                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
             headers
@@ -930,9 +917,7 @@ impl<'a> MimeFactory<'a> {
                     "ephemeral-timer-changed".to_string(),
                 ));
             }
-            SystemMessage::LocationOnly
-            | SystemMessage::MultiDeviceSync
-            | SystemMessage::WebxdcStatusUpdate => {
+            SystemMessage::LocationOnly | SystemMessage::MultiDeviceSync => {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
@@ -1133,7 +1118,7 @@ impl<'a> MimeFactory<'a> {
         }
 
         // add attachment part
-        if self.msg.viewtype.has_file() {
+        if chat::msgtype_has_file(self.msg.viewtype) {
             if !is_file_size_okay(context, self.msg).await? {
                 bail!(
                     "Message exceeds the recommended {} MB.",
@@ -1169,16 +1154,6 @@ impl<'a> MimeFactory<'a> {
             let ids = self.msg.param.get(Param::Arg2).unwrap_or_default();
             parts.push(context.build_sync_part(json.to_string()).await);
             self.sync_ids_to_delete = Some(ids.to_string());
-        } else if command == SystemMessage::WebxdcStatusUpdate {
-            let json = self.msg.param.get(Param::Arg).unwrap_or_default();
-            parts.push(context.build_status_update_part(json).await);
-        } else if self.msg.viewtype == Viewtype::Webxdc {
-            if let Some(json) = context
-                .render_webxdc_status_update_object(self.msg.id, None)
-                .await?
-            {
-                parts.push(context.build_status_update_part(&json).await);
-            }
         }
 
         if self.attach_selfavatar {
@@ -1310,7 +1285,7 @@ async fn build_body_file(
         .param
         .get_blob(Param::File, context, true)
         .await?
-        .context("msg has no filename")?;
+        .ok_or_else(|| format_err!("msg has no filename"))?;
     let suffix = blob.suffix().unwrap_or("dat");
 
     // Get file name to use for sending.  For privacy purposes, we do
@@ -1338,7 +1313,8 @@ async fn build_body_file(
             "video_{}.{}",
             chrono::Utc
                 .timestamp(msg.timestamp_sort, 0)
-                .format("%Y-%m-%d_%H-%M-%S"),
+                .format("%Y-%m-%d_%H-%M-%S")
+                .to_string(),
             &suffix
         ),
         _ => blob.as_file_name().to_string(),
@@ -1445,13 +1421,12 @@ fn maybe_encode_words(words: &str) -> String {
 
 #[cfg(test)]
 mod tests {
-    use async_std::fs::File;
+    use super::*;
     use async_std::prelude::*;
-    use mailparse::{addrparse_header, MailHeaderMap};
 
     use crate::chat::ChatId;
     use crate::chat::{
-        self, add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
+        add_contact_to_chat, create_group_chat, remove_contact_from_chat, send_text_msg,
         ProtectionStatus,
     };
     use crate::chatlist::Chatlist;
@@ -1460,7 +1435,9 @@ mod tests {
     use crate::mimeparser::MimeMessage;
     use crate::test_utils::{get_chat_msg, TestContext};
 
-    use super::*;
+    use async_std::fs::File;
+    use mailparse::{addrparse_header, MailHeaderMap};
+
     #[test]
     fn test_render_email_address() {
         let display_name = "ä space";
@@ -1665,6 +1642,7 @@ mod tests {
             Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
             \n\
             hello\n",
+            "INBOX",
             false,
         )
         .await
@@ -1710,7 +1688,7 @@ mod tests {
             let mut new_msg = Message::new(Viewtype::Text);
             new_msg.set_text(Some("Hi".to_string()));
             if let Some(q) = quote {
-                new_msg.set_quote(t, Some(q)).await?;
+                new_msg.set_quote(t, q).await?;
             }
             let sent = t.send_msg(group_id, &mut new_msg).await;
             get_subject(t, sent).await
@@ -1757,6 +1735,7 @@ mod tests {
                 t.get_last_msg().await.rfc724_mid
             )
             .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -1867,6 +1846,7 @@ mod tests {
                     Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
                     \n\
                     Some other, completely unrelated content\n",
+                "INBOX",
                 false,
             )
             .await
@@ -1877,7 +1857,7 @@ mod tests {
         }
 
         if reply {
-            new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap();
+            new_msg.set_quote(&t, &incoming_msg).await.unwrap();
         }
 
         let mf = MimeFactory::from_msg(&t, &new_msg, false).await.unwrap();
@@ -1891,11 +1871,13 @@ mod tests {
             .await
             .unwrap();
 
-        dc_receive_imf(context, imf_raw, false).await.unwrap();
+        dc_receive_imf(context, imf_raw, "INBOX", false)
+            .await
+            .unwrap();
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
-        let chat_id = chats.get_chat_id(0).unwrap();
+        let chat_id = chats.get_chat_id(0);
         chat_id.accept(context).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
@@ -1935,7 +1917,7 @@ mod tests {
 
         let rendered_msg = mimefactory.render(context).await.unwrap();
 
-        let mail = mailparse::parse_mail(rendered_msg.message.as_bytes()).unwrap();
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
         assert_eq!(
             mail.headers
                 .iter()
@@ -1945,7 +1927,7 @@ mod tests {
             "1.0"
         );
 
-        let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes())
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
             .await
             .unwrap();
     }
@@ -2019,9 +2001,8 @@ mod tests {
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("this is the text!".to_string()));
 
-        let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(3, "\r\n\r\n");
-
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(3, "\r\n\r\n");
         let outer = payload.next().unwrap();
         let inner = payload.next().unwrap();
         let body = payload.next().unwrap();
@@ -2039,8 +2020,8 @@ mod tests {
 
         // if another message is sent, that one must not contain the avatar
         // and no artificial multipart/mixed nesting
-        let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let mut payload = sent_msg.payload().splitn(2, "\r\n\r\n");
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(2, "\r\n\r\n");
         let outer = payload.next().unwrap();
         let body = payload.next().unwrap();
 
@@ -2079,36 +2060,13 @@ mod tests {
         let to = parsed
             .headers
             .get_first_header("To")
-            .context("no To: header parsed")?;
+            .ok_or_else(|| format_err!("No To: header parsed"))?;
         let to = addrparse_header(to)?;
         let mailbox = to
             .extract_single_info()
-            .context("to: field does not contain exactly one address")?;
+            .ok_or_else(|| format_err!("To: field does not contain exactly one address"))?;
         assert_eq!(mailbox.addr, "bob@example.net");
 
         Ok(())
     }
-
-    /// Tests that standard IMF header "From:" comes before non-standard "Autocrypt:" header.
-    #[async_std::test]
-    async fn test_from_before_autocrypt() -> Result<()> {
-        // create chat with bob
-        let t = TestContext::new_alice().await;
-        let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
-
-        // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
-        // make sure, `Subject:` stays in the outer header (imf header)
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("this is the text!".to_string()));
-
-        let sent_msg = t.send_msg(chat.id, &mut msg).await;
-        let payload = sent_msg.payload();
-
-        assert_eq!(payload.match_indices("Autocrypt:").count(), 1);
-        assert_eq!(payload.match_indices("From:").count(), 1);
-
-        assert!(payload.match_indices("From:").next() < payload.match_indices("Autocrypt:").next());
-
-        Ok(())
-    }
 }

src/mimeparser.rs

@@ -2,7 +2,6 @@
 
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
-use std::io::Cursor;
 use std::pin::Pin;
 
 use anyhow::{bail, Result};
@@ -13,8 +12,8 @@ use once_cell::sync::Lazy;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
-use crate::contact::{addr_normalize, ContactId};
+use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
+use crate::contact::addr_normalize;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_filemeta, dc_truncate, parse_receive_headers};
 use crate::dehtml::dehtml;
@@ -24,7 +23,7 @@ use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::Fingerprint;
 use crate::location;
-use crate::message::{self, Viewtype};
+use crate::message;
 use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
 use crate::simplify::simplify;
@@ -67,7 +66,6 @@ pub struct MimeMessage {
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
     pub(crate) sync_items: Option<SyncItems>,
-    pub(crate) webxdc_status_update: Option<String>,
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
@@ -137,10 +135,6 @@ pub enum SystemMessage {
     /// Self-sent-message that contains only json used for multi-device-sync;
     /// if possible, we attach that to other messages as for locations.
     MultiDeviceSync = 20,
-
-    // Sync message that contains a json payload
-    // sent to the other webxdc instances
-    WebxdcStatusUpdate = 30,
 }
 
 impl Default for SystemMessage {
@@ -302,7 +296,6 @@ impl MimeMessage {
             location_kml: None,
             message_kml: None,
             sync_items: None,
-            webxdc_status_update: None,
             user_avatar: None,
             group_avatar: None,
             failure_report: None,
@@ -369,16 +362,6 @@ impl MimeMessage {
             } else if value == "protection-disabled" {
                 self.is_system_message = SystemMessage::ChatProtectionDisabled;
             }
-        } else if self.get_header(HeaderDef::ChatGroupMemberRemoved).is_some() {
-            self.is_system_message = SystemMessage::MemberRemovedFromGroup;
-        } else if self.get_header(HeaderDef::ChatGroupMemberAdded).is_some() {
-            self.is_system_message = SystemMessage::MemberAddedToGroup;
-        } else if self.get_header(HeaderDef::ChatGroupNameChanged).is_some() {
-            self.is_system_message = SystemMessage::GroupNameChanged;
-        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
-            if value == "group-avatar-changed" {
-                self.is_system_message = SystemMessage::GroupImageChanged;
-            }
         }
     }
 
@@ -414,18 +397,16 @@ impl MimeMessage {
     #[allow(clippy::indexing_slicing)]
     fn squash_attachment_parts(&mut self) {
         if let [textpart, filepart] = &self.parts[..] {
-            let need_drop = textpart.typ == Viewtype::Text
-                && match filepart.typ {
-                    Viewtype::Image
-                    | Viewtype::Gif
-                    | Viewtype::Sticker
-                    | Viewtype::Audio
-                    | Viewtype::Voice
-                    | Viewtype::Video
-                    | Viewtype::File
-                    | Viewtype::Webxdc => true,
-                    Viewtype::Unknown | Viewtype::Text | Viewtype::VideochatInvitation => false,
-                };
+            let need_drop = {
+                textpart.typ == Viewtype::Text
+                    && (filepart.typ == Viewtype::Image
+                        || filepart.typ == Viewtype::Gif
+                        || filepart.typ == Viewtype::Sticker
+                        || filepart.typ == Viewtype::Audio
+                        || filepart.typ == Viewtype::Voice
+                        || filepart.typ == Viewtype::Video
+                        || filepart.typ == Viewtype::File)
+            };
 
             if need_drop {
                 let mut filepart = self.parts.swap_remove(1);
@@ -557,7 +538,7 @@ impl MimeMessage {
             };
 
             if let Some(ref subject) = self.get_subject() {
-                if !self.has_chat_version() && self.webxdc_status_update.is_none() {
+                if !self.has_chat_version() {
                     part.msg = subject.to_string();
                 }
             }
@@ -856,12 +837,6 @@ impl MimeMessage {
                                     .await?;
                             }
                         }
-                        Some("status-update") => {
-                            if let Some(second) = mail.subparts.get(1) {
-                                self.add_single_part_if_known(context, second, is_related)
-                                    .await?;
-                            }
-                        }
                         Some(_) => {
                             if let Some(first) = mail.subparts.get(0) {
                                 any_part_added = self
@@ -1031,14 +1006,8 @@ impl MimeMessage {
         if decoded_data.is_empty() {
             return;
         }
-        let reader = Cursor::new(decoded_data);
-        let msg_type = if context
-            .is_webxdc_file(filename, reader)
-            .await
-            .unwrap_or(false)
-        {
-            Viewtype::Webxdc
-        } else if filename.ends_with(".kml") {
+        // treat location/message kml file attachments specially
+        if filename.ends_with(".kml") {
             // XXX what if somebody sends eg an "location-highlights.kml"
             // attachment unrelated to location streaming?
             if filename.starts_with("location") || filename.starts_with("message") {
@@ -1054,7 +1023,6 @@ impl MimeMessage {
                 }
                 return;
             }
-            msg_type
         } else if filename == "multi-device-sync.json" {
             let serialized = String::from_utf8_lossy(decoded_data)
                 .parse()
@@ -1067,15 +1035,7 @@ impl MimeMessage {
                 })
                 .ok();
             return;
-        } else if filename == "status-update.json" {
-            let serialized = String::from_utf8_lossy(decoded_data)
-                .parse()
-                .unwrap_or_default();
-            self.webxdc_status_update = Some(serialized);
-            return;
-        } else {
-            msg_type
-        };
+        }
 
         /* we have a regular file attachment,
         write decoded data to new blob object */
@@ -1150,7 +1110,7 @@ impl MimeMessage {
         }
     }
 
-    pub(crate) fn get_rfc724_mid(&self) -> Option<String> {
+    pub fn get_rfc724_mid(&self) -> Option<String> {
         self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
             .or_else(|| self.get_header(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
@@ -1215,9 +1175,6 @@ impl MimeMessage {
         if let Some(_disposition) = report_fields.get_header_value(HeaderDef::Disposition) {
             let original_message_id = report_fields
                 .get_header_value(HeaderDef::OriginalMessageId)
-                // MS Exchange doesn't add an Original-Message-Id header. Instead, they put
-                // the original message id into the In-Reply-To header:
-                .or_else(|| report.headers.get_header_value(HeaderDef::InReplyTo))
                 .and_then(|v| parse_message_id(&v).ok());
             let additional_message_ids = report_fields
                 .get_header_value(HeaderDef::AdditionalMessageIds)
@@ -1381,7 +1338,7 @@ impl MimeMessage {
     pub async fn handle_reports(
         &self,
         context: &Context,
-        from_id: ContactId,
+        from_id: u32,
         sent_timestamp: i64,
         parts: &[Part],
     ) {
@@ -1495,8 +1452,8 @@ async fn update_gossip_peerstates(
 pub(crate) struct Report {
     /// Original-Message-ID header
     ///
-    /// It MUST be present if the original message has a Message-ID according to RFC 8098.
-    /// In case we can't find it (shouldn't happen), this is None.
+    /// It MUST be present if the original message has a Message-ID according to RFC 8098, but MS
+    /// Exchange does not add it nevertheless, in which case it is `None`.
     original_message_id: Option<String>,
     /// Additional-Message-IDs
     additional_message_ids: Vec<String>,
@@ -2915,6 +2872,7 @@ On 2020-10-25, Bob wrote:
         dc_receive_imf(
             &t.ctx,
             include_bytes!("../test-data/message/subj_with_multimedia_msg.eml"),
+            "INBOX",
             false,
         )
         .await
@@ -3063,7 +3021,7 @@ Subject: ...
 
 Some quote.
 "###;
-        dc_receive_imf(&t, raw, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
         let raw = br###"In-Reply-To:
@@ -3080,7 +3038,7 @@ Subject: ...
 Some reply
 "###;
 
-        dc_receive_imf(&t, raw, false).await?;
+        dc_receive_imf(&t, raw, "INBOX", false).await?;
 
         let msg = t.get_last_msg().await;
         assert_eq!(msg.get_text().unwrap(), "Some reply");
@@ -3108,13 +3066,13 @@ Message.
 "###;
 
         // Bob receives message.
-        dc_receive_imf(&bob, raw, false).await?;
+        dc_receive_imf(&bob, raw, "INBOX", false).await?;
         let msg = bob.get_last_msg().await;
         // Message is incoming.
         assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
 
         // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, false).await?;
+        dc_receive_imf(&alice, raw, "INBOX", false).await?;
         let msg = alice.get_last_msg().await;
         // Message is outgoing, don't send read receipt to self.
         assert!(msg.param.get_bool(Param::WantsMdn).is_none());
@@ -3140,6 +3098,7 @@ Message.
                  \n\
                  hello\n"
                 .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -3177,6 +3136,7 @@ Message.
                  \n\
                  --SNIPP--"
             .as_bytes(),
+            "INBOX",
             false,
         )
         .await?;
@@ -3195,32 +3155,10 @@ Message.
     #[async_std::test]
     async fn test_ms_exchange_mdn() -> Result<()> {
         let t = TestContext::new_alice().await;
-        t.set_config(Config::ShowEmails, Some("2")).await?;
-
-        let original =
-            include_bytes!("../test-data/message/ms_exchange_report_original_message.eml");
-        dc_receive_imf(&t, original, false).await?;
-        let original_msg_id = t.get_last_msg().await.id;
-
-        // 1. Test mimeparser directly
-        let mdn =
+        let raw =
             include_bytes!("../test-data/message/ms_exchange_report_disposition_notification.eml");
-        let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?;
-        assert_eq!(mimeparser.mdn_reports.len(), 1);
-        assert_eq!(
-            mimeparser.mdn_reports[0].original_message_id.as_deref(),
-            Some("d5904dc344eeb5deaf9bb44603f0c716@posteo.de")
-        );
-        assert!(mimeparser.mdn_reports[0].additional_message_ids.is_empty());
-
-        // 2. Test that marking the original msg as read works
-        dc_receive_imf(&t, mdn, false).await?;
-
-        assert_eq!(
-            original_msg_id.get_state(&t).await?,
-            MessageState::OutMdnRcvd
-        );
-
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await?;
+        assert!(!mimeparser.mdn_reports.is_empty());
         Ok(())
     }
 }

src/param.rs

@@ -110,6 +110,9 @@ pub enum Param {
     /// For Jobs
     AlsoMove = b'M',
 
+    /// For Jobs: space-separated list of message recipients
+    Recipients = b'R',
+
     /// For MDN-sending job
     MsgId = b'I',
 
@@ -168,12 +171,6 @@ pub enum Param {
 
     /// For Chats: timestamp of protection settings update.
     ProtectionSettingsTimestamp = b'L',
-
-    /// For Webxdc Message Instances: Current summary
-    WebxdcSummary = b'N',
-
-    /// For Webxdc Message Instances: timestamp of summary update.
-    WebxdcSummaryTimestamp = b'Q',
 }
 
 /// An object for handling key=value parameter lists.

src/peerstate.rs

@@ -4,13 +4,11 @@ use std::collections::HashSet;
 use std::fmt;
 
 use crate::aheader::{Aheader, EncryptPreference};
-use crate::chat::{self};
-use crate::chatlist::Chatlist;
+use crate::chat::{self, ChatIdBlocked};
+use crate::constants::Blocked;
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::message::Message;
-use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
 use anyhow::{bail, Result};
@@ -273,34 +271,14 @@ impl Peerstate {
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chats = Chatlist::try_load(context, 0, None, contact_id).await?;
+                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
+                    .await?
+                    .id;
+
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
-                for (chat_id, msg_id) in chats.iter() {
-                    let timestamp_sort = if let Some(msg_id) = msg_id {
-                        let lastmsg = Message::load_from_db(context, *msg_id).await?;
-                        lastmsg.timestamp_sort
-                    } else {
-                        context
-                            .sql
-                            .query_get_value(
-                                "SELECT created_timestamp FROM chats WHERE id=?;",
-                                paramsv![chat_id],
-                            )
-                            .await?
-                            .unwrap_or(0)
-                    };
-                    chat::add_info_msg_with_cmd(
-                        context,
-                        *chat_id,
-                        &msg,
-                        SystemMessage::Unknown,
-                        timestamp_sort,
-                        Some(timestamp),
-                        None,
-                    )
-                    .await?;
-                    context.emit_event(EventType::ChatModified(*chat_id));
-                }
+
+                chat::add_info_msg(context, chat_id, &msg, timestamp).await?;
+                context.emit_event(EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -401,9 +379,10 @@ impl Peerstate {
     pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&SignedPublicKey> {
         match min_verified {
             PeerstateVerifiedStatus::BidirectVerified => self.verified_key.as_ref(),
-            PeerstateVerifiedStatus::Unverified => {
-                self.public_key.as_ref().or(self.gossip_key.as_ref())
-            }
+            PeerstateVerifiedStatus::Unverified => self
+                .public_key
+                .as_ref()
+                .or_else(|| self.gossip_key.as_ref()),
         }
     }
 

src/pgp.rs

@@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
-use anyhow::{bail, ensure, format_err, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -346,7 +346,7 @@ pub async fn pk_validate(
     // OpenPGP signature calculation.
     let content = content
         .get(..content.len().saturating_sub(2))
-        .context("index is out of range")?;
+        .ok_or_else(|| format_err!("index is out of range"))?;
 
     for pkey in pkeys {
         if standalone_signature.verify(pkey, content).is_ok() {
@@ -520,6 +520,7 @@ mod tests {
             &sig_check_keyring,
         )
         .await
+        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);
@@ -535,6 +536,7 @@ mod tests {
             &sig_check_keyring,
         )
         .await
+        .map_err(|err| println!("{:?}", err))
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
         assert_eq!(valid_signatures.len(), 1);

src/plaintext.rs

@@ -41,7 +41,7 @@ impl PlainText {
             // as <http://example.org> cannot be handled correctly
             // (they would become &lt;http://example.org&gt; where the trailing &gt; would become a valid url part).
             // to avoid double encoding, we escape our html-entities by \r that must not be used in the string elsewhere.
-            let line = line.to_string().replace('\r', "");
+            let line = line.to_string().replace("\r", "");
 
             let mut line = LINKIFY_MAIL_RE
                 .replace_all(&*line, "\rLTa href=\rQUOTmailto:$1\rQUOT\rGT$1\rLT/a\rGT")

src/provider.rs

@@ -113,7 +113,7 @@ pub async fn get_provider_info(
     domain: &str,
     skip_mx: bool,
 ) -> Option<&'static Provider> {
-    let domain = domain.rsplit('@').next()?;
+    let domain = domain.rsplitn(2, '@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
         return Some(provider);

src/provider/data.rs

@@ -44,7 +44,7 @@ static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
             protocol: Smtp,
             socket: Starttls,
             hostname: "newyear.aktivix.org",
-            port: 587,
+            port: 25,
             username_pattern: Email,
         },
     ],
@@ -282,23 +282,57 @@ static P_DISROOT: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/disroot",
+    server: vec![],
+    config_defaults: None,
+    strict_tls: true,
+    max_smtp_rcpt_to: None,
+    oauth2_authorizer: None,
+});
+
+// dubby.org.md: dubby.org
+static P_DUBBY_ORG: Lazy<Provider> = Lazy::new(|| Provider {
+    id: "dubby.org",
+    status: Status::Ok,
+    before_login_hint: "",
+    after_login_hint: "",
+    overview_page: "https://providers.delta.chat/dubby-org",
     server: vec![
         Server {
             protocol: Imap,
             socket: Ssl,
-            hostname: "disroot.org",
+            hostname: "dubby.org",
             port: 993,
             username_pattern: Email,
         },
         Server {
             protocol: Smtp,
             socket: Starttls,
-            hostname: "disroot.org",
+            hostname: "dubby.org",
             port: 587,
             username_pattern: Email,
         },
+        Server {
+            protocol: Smtp,
+            socket: Ssl,
+            hostname: "dubby.org",
+            port: 465,
+            username_pattern: Email,
+        },
     ],
-    config_defaults: None,
+    config_defaults: Some(vec![
+        ConfigDefault {
+            key: Config::BccSelf,
+            value: "1",
+        },
+        ConfigDefault {
+            key: Config::SentboxWatch,
+            value: "0",
+        },
+        ConfigDefault {
+            key: Config::MvboxMove,
+            value: "0",
+        },
+    ]),
     strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
@@ -366,7 +400,7 @@ static P_EXAMPLE_COM: Lazy<Provider> = Lazy::new(|| {
 }
 });
 
-// fastmail.md: 123mail.org, 150mail.com, 150ml.com, 16mail.com, 2-mail.com, 4email.net, 50mail.com, airpost.net, allmail.net, bestmail.us, cluemail.com, elitemail.org, emailcorner.net, emailengine.net, emailengine.org, emailgroups.net, emailplus.org, emailuser.net, eml.cc, f-m.fm, fast-email.com, fast-mail.org, fastem.com, fastemail.us, fastemailer.com, fastest.cc, fastimap.com, fastmail.cn, fastmail.co.uk, fastmail.com, fastmail.com.au, fastmail.de, fastmail.es, fastmail.fm, fastmail.fr, fastmail.im, fastmail.in, fastmail.jp, fastmail.mx, fastmail.net, fastmail.nl, fastmail.org, fastmail.se, fastmail.to, fastmail.tw, fastmail.uk, fastmail.us, fastmailbox.net, fastmessaging.com, fea.st, fmail.co.uk, fmailbox.com, fmgirl.com, fmguy.com, ftml.net, h-mail.us, hailmail.net, imap-mail.com, imap.cc, imapmail.org, inoutbox.com, internet-e-mail.com, internet-mail.org, internetemails.net, internetmailing.net, jetemail.net, justemail.net, letterboxes.org, mail-central.com, mail-page.com, mailandftp.com, mailas.com, mailbolt.com, mailc.net, mailcan.com, mailforce.net, mailftp.com, mailhaven.com, mailingaddress.org, mailite.com, mailmight.com, mailnew.com, mailsent.net, mailservice.ms, mailup.net, mailworks.org, ml1.net, mm.st, myfastmail.com, mymacmail.com, nospammail.net, ownmail.net, petml.com, postinbox.com, postpro.net, proinbox.com, promessage.com, realemail.net, reallyfast.biz, reallyfast.info, rushpost.com, sent.as, sent.at, sent.com, speedpost.net, speedymail.org, ssl-mail.com, swift-mail.com, the-fastest.net, the-quickest.com, theinternetemail.com, veryfast.biz, veryspeedy.net, warpmail.net, xsmail.com, yepmail.net, your-mail.com
+// fastmail.md: fastmail.com
 static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
     id: "fastmail",
     status: Status::Preparation,
@@ -389,6 +423,13 @@ static P_FASTMAIL: Lazy<Provider> = Lazy::new(|| Provider {
             port: 465,
             username_pattern: Email,
         },
+        Server {
+            protocol: Smtp,
+            socket: Starttls,
+            hostname: "smtp.fastmail.com",
+            port: 587,
+            username_pattern: Email,
+        },
     ],
     config_defaults: None,
     strict_tls: true,
@@ -619,35 +660,6 @@ static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// infomaniak.com.md: ik.me
-static P_INFOMANIAK_COM: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "infomaniak.com",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/infomaniak-com",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.infomaniak.com",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.infomaniak.com",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: Some(10),
-    oauth2_authorizer: None,
-});
-
 // kolst.com.md: kolst.com
 static P_KOLST_COM: Lazy<Provider> = Lazy::new(|| Provider {
     id: "kolst.com",
@@ -676,41 +688,12 @@ static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// mail.de.md: mail.de
-static P_MAIL_DE: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "mail.de",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/mail-de",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.mail.de",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "smtp.mail.de",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // mail.ru.md: mail.ru, inbox.ru, internet.ru, bk.ru, list.ru
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
     Provider {
     id: "mail.ru",
-    status: Status::Preparation,
-    before_login_hint: "Вам необходимо сгенерировать \"пароль для внешнего приложения\" в веб-интерфейсе mail.ru, чтобы mail.ru работал с Delta Chat.",
+    status: Status::Ok,
+    before_login_hint: "Не рекомендуется использовать mail.ru, потому что он разряжает вашу батарею быстрее, чем другие провайдеры.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mail-ru",
     server: vec![
@@ -760,22 +743,7 @@ static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mailbox-org",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.mailbox.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "smtp.mailbox.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -851,6 +819,10 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
             key: Config::MvboxMove,
             value: "0",
         },
+        ConfigDefault {
+            key: Config::E2eeEnabled,
+            value: "0",
+        },
         ConfigDefault {
             key: Config::MediaQuality,
             value: "1",
@@ -894,7 +866,7 @@ static P_NAVER: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com, outlook.de
+// outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
 static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     id: "outlook.com",
     status: Status::Ok,
@@ -923,7 +895,7 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ca, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
+// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
 static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "posteo",
     status: Status::Ok,
@@ -952,7 +924,7 @@ static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// protonmail.md: protonmail.com, protonmail.ch, pm.me
+// protonmail.md: protonmail.com, protonmail.ch
 static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
     Provider {
     id: "protonmail",
@@ -979,7 +951,7 @@ static P_QQ: Lazy<Provider> = Lazy::new(|| {
     overview_page: "https://providers.delta.chat/qq",
     server: vec![
         Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart },
-        Server { protocol: Smtp, socket: Ssl, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
+        Server { protocol: Smtp, socket: Starttls, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
     ],
     config_defaults: None,
     strict_tls: true,
@@ -995,22 +967,7 @@ static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/riseup-net",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.riseup.net",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.riseup.net",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -1067,22 +1024,7 @@ static P_SYSTEMLI_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/systemli-org",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "mail.systemli.org",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "mail.systemli.org",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
+    server: vec![],
     config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
@@ -1398,9 +1340,7 @@ static P_YGGMAIL: Lazy<Provider> = Lazy::new(|| {
         Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email },
         Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email },
     ],
-    config_defaults: Some(vec![
-        ConfigDefault { key: Config::MvboxMove, value: "0" },
-    ]),
+    config_defaults: None,
     strict_tls: true,
     max_smtp_rcpt_to: None,
     oauth2_authorizer: None,
@@ -1481,128 +1421,13 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("comcast.net", &*P_COMCAST),
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot.org", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("e.email", &*P_E_EMAIL),
         ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("example.net", &*P_EXAMPLE_COM),
-        ("123mail.org", &*P_FASTMAIL),
-        ("150mail.com", &*P_FASTMAIL),
-        ("150ml.com", &*P_FASTMAIL),
-        ("16mail.com", &*P_FASTMAIL),
-        ("2-mail.com", &*P_FASTMAIL),
-        ("4email.net", &*P_FASTMAIL),
-        ("50mail.com", &*P_FASTMAIL),
-        ("airpost.net", &*P_FASTMAIL),
-        ("allmail.net", &*P_FASTMAIL),
-        ("bestmail.us", &*P_FASTMAIL),
-        ("cluemail.com", &*P_FASTMAIL),
-        ("elitemail.org", &*P_FASTMAIL),
-        ("emailcorner.net", &*P_FASTMAIL),
-        ("emailengine.net", &*P_FASTMAIL),
-        ("emailengine.org", &*P_FASTMAIL),
-        ("emailgroups.net", &*P_FASTMAIL),
-        ("emailplus.org", &*P_FASTMAIL),
-        ("emailuser.net", &*P_FASTMAIL),
-        ("eml.cc", &*P_FASTMAIL),
-        ("f-m.fm", &*P_FASTMAIL),
-        ("fast-email.com", &*P_FASTMAIL),
-        ("fast-mail.org", &*P_FASTMAIL),
-        ("fastem.com", &*P_FASTMAIL),
-        ("fastemail.us", &*P_FASTMAIL),
-        ("fastemailer.com", &*P_FASTMAIL),
-        ("fastest.cc", &*P_FASTMAIL),
-        ("fastimap.com", &*P_FASTMAIL),
-        ("fastmail.cn", &*P_FASTMAIL),
-        ("fastmail.co.uk", &*P_FASTMAIL),
         ("fastmail.com", &*P_FASTMAIL),
-        ("fastmail.com.au", &*P_FASTMAIL),
-        ("fastmail.de", &*P_FASTMAIL),
-        ("fastmail.es", &*P_FASTMAIL),
-        ("fastmail.fm", &*P_FASTMAIL),
-        ("fastmail.fr", &*P_FASTMAIL),
-        ("fastmail.im", &*P_FASTMAIL),
-        ("fastmail.in", &*P_FASTMAIL),
-        ("fastmail.jp", &*P_FASTMAIL),
-        ("fastmail.mx", &*P_FASTMAIL),
-        ("fastmail.net", &*P_FASTMAIL),
-        ("fastmail.nl", &*P_FASTMAIL),
-        ("fastmail.org", &*P_FASTMAIL),
-        ("fastmail.se", &*P_FASTMAIL),
-        ("fastmail.to", &*P_FASTMAIL),
-        ("fastmail.tw", &*P_FASTMAIL),
-        ("fastmail.uk", &*P_FASTMAIL),
-        ("fastmail.us", &*P_FASTMAIL),
-        ("fastmailbox.net", &*P_FASTMAIL),
-        ("fastmessaging.com", &*P_FASTMAIL),
-        ("fea.st", &*P_FASTMAIL),
-        ("fmail.co.uk", &*P_FASTMAIL),
-        ("fmailbox.com", &*P_FASTMAIL),
-        ("fmgirl.com", &*P_FASTMAIL),
-        ("fmguy.com", &*P_FASTMAIL),
-        ("ftml.net", &*P_FASTMAIL),
-        ("h-mail.us", &*P_FASTMAIL),
-        ("hailmail.net", &*P_FASTMAIL),
-        ("imap-mail.com", &*P_FASTMAIL),
-        ("imap.cc", &*P_FASTMAIL),
-        ("imapmail.org", &*P_FASTMAIL),
-        ("inoutbox.com", &*P_FASTMAIL),
-        ("internet-e-mail.com", &*P_FASTMAIL),
-        ("internet-mail.org", &*P_FASTMAIL),
-        ("internetemails.net", &*P_FASTMAIL),
-        ("internetmailing.net", &*P_FASTMAIL),
-        ("jetemail.net", &*P_FASTMAIL),
-        ("justemail.net", &*P_FASTMAIL),
-        ("letterboxes.org", &*P_FASTMAIL),
-        ("mail-central.com", &*P_FASTMAIL),
-        ("mail-page.com", &*P_FASTMAIL),
-        ("mailandftp.com", &*P_FASTMAIL),
-        ("mailas.com", &*P_FASTMAIL),
-        ("mailbolt.com", &*P_FASTMAIL),
-        ("mailc.net", &*P_FASTMAIL),
-        ("mailcan.com", &*P_FASTMAIL),
-        ("mailforce.net", &*P_FASTMAIL),
-        ("mailftp.com", &*P_FASTMAIL),
-        ("mailhaven.com", &*P_FASTMAIL),
-        ("mailingaddress.org", &*P_FASTMAIL),
-        ("mailite.com", &*P_FASTMAIL),
-        ("mailmight.com", &*P_FASTMAIL),
-        ("mailnew.com", &*P_FASTMAIL),
-        ("mailsent.net", &*P_FASTMAIL),
-        ("mailservice.ms", &*P_FASTMAIL),
-        ("mailup.net", &*P_FASTMAIL),
-        ("mailworks.org", &*P_FASTMAIL),
-        ("ml1.net", &*P_FASTMAIL),
-        ("mm.st", &*P_FASTMAIL),
-        ("myfastmail.com", &*P_FASTMAIL),
-        ("mymacmail.com", &*P_FASTMAIL),
-        ("nospammail.net", &*P_FASTMAIL),
-        ("ownmail.net", &*P_FASTMAIL),
-        ("petml.com", &*P_FASTMAIL),
-        ("postinbox.com", &*P_FASTMAIL),
-        ("postpro.net", &*P_FASTMAIL),
-        ("proinbox.com", &*P_FASTMAIL),
-        ("promessage.com", &*P_FASTMAIL),
-        ("realemail.net", &*P_FASTMAIL),
-        ("reallyfast.biz", &*P_FASTMAIL),
-        ("reallyfast.info", &*P_FASTMAIL),
-        ("rushpost.com", &*P_FASTMAIL),
-        ("sent.as", &*P_FASTMAIL),
-        ("sent.at", &*P_FASTMAIL),
-        ("sent.com", &*P_FASTMAIL),
-        ("speedpost.net", &*P_FASTMAIL),
-        ("speedymail.org", &*P_FASTMAIL),
-        ("ssl-mail.com", &*P_FASTMAIL),
-        ("swift-mail.com", &*P_FASTMAIL),
-        ("the-fastest.net", &*P_FASTMAIL),
-        ("the-quickest.com", &*P_FASTMAIL),
-        ("theinternetemail.com", &*P_FASTMAIL),
-        ("veryfast.biz", &*P_FASTMAIL),
-        ("veryspeedy.net", &*P_FASTMAIL),
-        ("warpmail.net", &*P_FASTMAIL),
-        ("xsmail.com", &*P_FASTMAIL),
-        ("yepmail.net", &*P_FASTMAIL),
-        ("your-mail.com", &*P_FASTMAIL),
         ("firemail.at", &*P_FIREMAIL_DE),
         ("firemail.de", &*P_FIREMAIL_DE),
         ("five.chat", &*P_FIVE_CHAT),
@@ -1626,10 +1451,8 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
-        ("ik.me", &*P_INFOMANIAK_COM),
         ("kolst.com", &*P_KOLST_COM),
         ("kontent.com", &*P_KONTENT_COM),
-        ("mail.de", &*P_MAIL_DE),
         ("mail.ru", &*P_MAIL_RU),
         ("inbox.ru", &*P_MAIL_RU),
         ("internet.ru", &*P_MAIL_RU),
@@ -1646,12 +1469,10 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("office365.com", &*P_OUTLOOK_COM),
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
-        ("outlook.de", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
         ("posteo.af", &*P_POSTEO),
         ("posteo.at", &*P_POSTEO),
         ("posteo.be", &*P_POSTEO),
-        ("posteo.ca", &*P_POSTEO),
         ("posteo.ch", &*P_POSTEO),
         ("posteo.cl", &*P_POSTEO),
         ("posteo.co", &*P_POSTEO),
@@ -1700,7 +1521,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
-        ("pm.me", &*P_PROTONMAIL),
         ("qq.com", &*P_QQ),
         ("foxmail.com", &*P_QQ),
         ("riseup.net", &*P_RISEUP_NET),
@@ -1797,6 +1617,7 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("comcast", &*P_COMCAST),
         ("dismail.de", &*P_DISMAIL_DE),
         ("disroot", &*P_DISROOT),
+        ("dubby.org", &*P_DUBBY_ORG),
         ("e.email", &*P_E_EMAIL),
         ("espiv.net", &*P_ESPIV_NET),
         ("example.com", &*P_EXAMPLE_COM),
@@ -1811,10 +1632,8 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("i.ua", &*P_I_UA),
         ("i3.net", &*P_I3_NET),
         ("icloud", &*P_ICLOUD),
-        ("infomaniak.com", &*P_INFOMANIAK_COM),
         ("kolst.com", &*P_KOLST_COM),
         ("kontent.com", &*P_KONTENT_COM),
-        ("mail.de", &*P_MAIL_DE),
         ("mail.ru", &*P_MAIL_RU),
         ("mail2tor", &*P_MAIL2TOR),
         ("mailbox.org", &*P_MAILBOX_ORG),
@@ -1851,4 +1670,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2022, 5, 3));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 9, 29));

src/qr.rs

@@ -9,7 +9,7 @@ use std::collections::BTreeMap;
 use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::Blocked;
-use crate::contact::{addr_normalize, may_be_valid_addr, Contact, ContactId, Origin};
+use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::key::Fingerprint;
@@ -30,7 +30,7 @@ const HTTPS_SCHEME: &str = "https://";
 #[derive(Debug, Clone, PartialEq)]
 pub enum Qr {
     AskVerifyContact {
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -38,16 +38,16 @@ pub enum Qr {
     AskVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     FprOk {
-        contact_id: ContactId,
+        contact_id: u32,
     },
     FprMismatch {
-        contact_id: Option<ContactId>,
+        contact_id: Option<u32>,
     },
     FprWithoutAddr {
         fingerprint: String,
@@ -60,7 +60,7 @@ pub enum Qr {
         instance_pattern: String,
     },
     Addr {
-        contact_id: ContactId,
+        contact_id: u32,
     },
     Url {
         url: String,
@@ -69,7 +69,7 @@ pub enum Qr {
         text: String,
     },
     WithdrawVerifyContact {
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -77,13 +77,13 @@ pub enum Qr {
     WithdrawVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     ReviveVerifyContact {
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -91,7 +91,7 @@ pub enum Qr {
     ReviveVerifyGroup {
         grpname: String,
         grpid: String,
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
@@ -173,7 +173,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     };
 
     let name = if let Some(encoded_name) = param.get("n") {
-        let encoded_name = encoded_name.replace('+', "%20"); // sometimes spaces are encoded as `+`
+        let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
         match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
             Err(err) => bail!("Invalid name: {}", err),
@@ -188,7 +188,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get("g") {
-            let encoded_name = encoded_name.replace('+', "%20"); // sometimes spaces are encoded as `+`
+            let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
             match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
                 Err(err) => bail!("Invalid group name: {}", err),
@@ -304,14 +304,14 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
 fn decode_account(qr: &str) -> Result<Qr> {
     let payload = qr
         .get(DCACCOUNT_SCHEME.len()..)
-        .context("invalid DCACCOUNT payload")?;
+        .ok_or_else(|| format_err!("Invalid DCACCOUNT payload"))?;
     let url =
         url::Url::parse(payload).with_context(|| format!("Invalid account URL: {:?}", payload))?;
     if url.scheme() == "http" || url.scheme() == "https" {
         Ok(Qr::Account {
             domain: url
                 .host_str()
-                .context("can't extract WebRTC instance domain")?
+                .ok_or_else(|| format_err!("Can't extract WebRTC instance domain"))?
                 .to_string(),
         })
     } else {
@@ -323,7 +323,7 @@ fn decode_account(qr: &str) -> Result<Qr> {
 fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result<Qr> {
     let payload = qr
         .get(DCWEBRTC_SCHEME.len()..)
-        .context("invalid DCWEBRTC payload")?;
+        .ok_or_else(|| format_err!("Invalid DCWEBRTC payload"))?;
 
     let (_type, url) = Message::parse_webrtc_instance(payload);
     let url =
@@ -333,7 +333,7 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result<Qr> {
         Ok(Qr::WebrtcInstance {
             domain: url
                 .host_str()
-                .context("can't extract WebRTC instance domain")?
+                .ok_or_else(|| format_err!("Can't extract WebRTC instance domain"))?
                 .to_string(),
             instance_pattern: payload.to_string(),
         })
@@ -711,7 +711,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, 0);
             assert_eq!(grpname, "test ? test !");
         } else {
             bail!("Wrong QR code type");
@@ -729,7 +729,7 @@ mod tests {
             ..
         } = qr
         {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, 0);
             assert_eq!(grpname, "test ? test !");
 
             let contact = Contact::get_by_id(&ctx.ctx, contact_id).await?;
@@ -751,7 +751,7 @@ mod tests {
         ).await?;
 
         if let Qr::AskVerifyContact { contact_id, .. } = qr {
-            assert_ne!(contact_id, ContactId::UNDEFINED);
+            assert_ne!(contact_id, 0);
         } else {
             bail!("Wrong QR code type");
         }

src/qr_code_generator.rs

@@ -6,7 +6,8 @@ use crate::{
     chat::{Chat, ChatId},
     color::color_int_to_hex_string,
     config::Config,
-    contact::{Contact, ContactId},
+    constants::DC_CONTACT_ID_SELF,
+    contact::Contact,
     context::Context,
     securejoin, stock_str,
 };
@@ -40,7 +41,7 @@ async fn generate_join_group_qr_code(context: &Context, chat_id: ChatId) -> Resu
 }
 
 async fn generate_verification_qr(context: &Context) -> Result<String> {
-    let contact = Contact::get_by_id(context, ContactId::SELF).await?;
+    let contact = Contact::get_by_id(context, DC_CONTACT_ID_SELF).await?;
 
     let avatar = match contact.get_profile_image(context).await? {
         Some(path) => {
@@ -65,12 +66,13 @@ async fn generate_verification_qr(context: &Context) -> Result<String> {
 }
 
 fn inner_generate_secure_join_qr_code(
-    qrcode_description: &str,
+    raw_qrcode_description: &str,
     qrcode_content: &str,
     color: &str,
     avatar: Option<Vec<u8>>,
     avatar_letter: char,
 ) -> Result<String> {
+    let qrcode_description = &escaper::encode_minimal(raw_qrcode_description);
     // config
     let width = 515.0;
     let height = 630.0;
@@ -261,21 +263,3 @@ fn inner_generate_secure_join_qr_code(
 
     Ok(svg)
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[async_std::test]
-    async fn test_svg_escaping() {
-        let svg = inner_generate_secure_join_qr_code(
-            "descr123 \" < > &",
-            "qr-code-content",
-            "#000000",
-            None,
-            'X',
-        )
-        .unwrap();
-        assert!(svg.contains("descr123 &quot; &lt; &gt; &amp;"))
-    }
-}

src/quota.rs

@@ -1,17 +1,18 @@
 //! # Support for IMAP QUOTA extension.
 
-use anyhow::{anyhow, Context as _, Result};
+use anyhow::{anyhow, Result};
 use async_imap::types::{Quota, QuotaResource};
 use std::collections::BTreeMap;
 
 use crate::chat::add_device_msg_with_importance;
 use crate::config::Config;
+use crate::constants::Viewtype;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::Imap;
 use crate::job::{Action, Status};
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::param::Params;
 use crate::{job, stock_str, EventType};
 
@@ -63,7 +64,7 @@ async fn get_unique_quota_roots_and_usage(
                     .iter()
                     .find(|q| &q.root_name == quota_root_name)
                     .cloned()
-                    .context("quota_root should have a quota")?;
+                    .ok_or_else(|| anyhow!("quota_root should have a quota"))?;
                 // replace old quotas, because between fetching quotaroots for folders,
                 // messages could be recieved and so the usage could have been changed
                 *unique_quota_roots
@@ -95,7 +96,7 @@ fn get_highest_usage<'t>(
         }
     }
 
-    highest.context("no quota_resource found, this is unexpected")
+    highest.ok_or_else(|| anyhow!("no quota_resource found, this is unexpected"))
 }
 
 /// Checks if a quota warning is needed.

src/scheduler.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, Context as _, Result};
+use anyhow::{bail, Result};
 use async_std::prelude::*;
 use async_std::{
     channel::{self, Receiver, Sender},
@@ -8,19 +8,16 @@ use async_std::{
 use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::maybe_add_time_based_warnings;
-use crate::dc_tools::time;
-use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::imap::Imap;
 use crate::job::{self, Thread};
-use crate::location;
-use crate::log::LogExt;
-use crate::smtp::{send_smtp_messages, Smtp};
-use crate::sql;
+use crate::smtp::Smtp;
 
 use self::connectivity::ConnectivityStore;
 
 pub(crate) mod connectivity;
 
+pub(crate) struct StopToken;
+
 /// Job and connection scheduler.
 #[derive(Debug)]
 #[allow(clippy::large_enum_variant)]
@@ -35,10 +32,6 @@ pub(crate) enum Scheduler {
         sentbox_handle: Option<task::JoinHandle<()>>,
         smtp: SmtpConnectionState,
         smtp_handle: Option<task::JoinHandle<()>>,
-        ephemeral_handle: Option<task::JoinHandle<()>>,
-        ephemeral_interrupt_send: Sender<()>,
-        location_handle: Option<task::JoinHandle<()>>,
-        location_interrupt_send: Sender<()>,
     },
 }
 
@@ -64,14 +57,6 @@ impl Context {
     pub(crate) async fn interrupt_smtp(&self, info: InterruptInfo) {
         self.scheduler.read().await.interrupt_smtp(info).await;
     }
-
-    pub(crate) async fn interrupt_ephemeral_task(&self) {
-        self.scheduler.read().await.interrupt_ephemeral_task().await;
-    }
-
-    pub(crate) async fn interrupt_location(&self) {
-        self.scheduler.read().await.interrupt_location().await;
-    }
 }
 
 async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConnectionHandlers) {
@@ -81,16 +66,19 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
+        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
+        started
+            .send(())
+            .await
+            .expect("inbox loop, missing started receiver");
         let ctx = ctx1;
-        if let Err(err) = started.send(()).await {
-            warn!(ctx, "inbox loop, missing started receiver: {}", err);
-            return;
-        };
 
+        // track number of continously executed jobs
+        let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
             let job = match job::load_next(&ctx, Thread::Imap, &info).await {
@@ -102,25 +90,21 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
             };
 
             match job {
-                Some(job) => {
+                Some(job) if jobs_loaded <= 20 => {
+                    jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
                     info = Default::default();
                 }
+                Some(job) => {
+                    // Let the fetch run, but return back to the job afterwards.
+                    jobs_loaded = 0;
+                    info!(ctx, "postponing imap-job {} to run fetch...", job);
+                    fetch(&ctx, &mut connection).await;
+                }
                 None => {
-                    maybe_add_time_based_warnings(&ctx).await;
+                    jobs_loaded = 0;
 
-                    match ctx.get_config_i64(Config::LastHousekeeping).await {
-                        Ok(last_housekeeping_time) => {
-                            let next_housekeeping_time =
-                                last_housekeeping_time.saturating_add(60 * 60 * 24);
-                            if next_housekeeping_time <= time() {
-                                sql::housekeeping(&ctx).await.ok_or_log(&ctx);
-                            }
-                        }
-                        Err(err) => {
-                            warn!(ctx, "Failed to get last housekeeping time: {}", err);
-                        }
-                    };
+                    maybe_add_time_based_warnings(&ctx).await;
 
                     info = fetch_idle(&ctx, &mut connection, Config::ConfiguredInboxFolder).await;
                 }
@@ -135,6 +119,36 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         })
         .race(fut)
         .await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("inbox loop, missing shutdown receiver");
+}
+
+async fn fetch(ctx: &Context, connection: &mut Imap) {
+    match ctx.get_config(Config::ConfiguredInboxFolder).await {
+        Ok(Some(watch_folder)) => {
+            if let Err(err) = connection.prepare(ctx).await {
+                warn!(ctx, "Could not connect: {}", err);
+                return;
+            }
+
+            // fetch
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
+                connection.trigger_reconnect(ctx).await;
+                warn!(ctx, "{:#}", err);
+            }
+        }
+        Ok(None) => {
+            info!(ctx, "Can not fetch inbox folder, not set");
+        }
+        Err(err) => {
+            warn!(
+                ctx,
+                "Can not fetch inbox folder, failed to get config: {:?}", err
+            );
+        }
+    }
 }
 
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
@@ -146,76 +160,25 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 return connection.fake_idle(ctx, Some(watch_folder)).await;
             }
 
+            // Scan other folders before fetching from watched folder. This may result in the
+            // messages being moved into the watched folder, for example from the Spam folder to
+            // the Inbox folder.
             if folder == Config::ConfiguredInboxFolder {
-                if let Err(err) = connection
-                    .store_seen_flags_on_imap(ctx)
-                    .await
-                    .context("store_seen_flags_on_imap failed")
-                {
-                    warn!(ctx, "{:#}", err);
+                // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
+                if let Err(err) = connection.scan_folders(ctx).await {
+                    // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
+                    // but maybe just one folder can't be selected or something
+                    warn!(ctx, "{}", err);
                 }
             }
 
-            // Fetch the watched folder.
-            if let Err(err) = connection
-                .fetch_move_delete(ctx, &watch_folder, false)
-                .await
-            {
+            // fetch
+            if let Err(err) = connection.fetch_move_delete(ctx, &watch_folder).await {
                 connection.trigger_reconnect(ctx).await;
                 warn!(ctx, "{:#}", err);
                 return InterruptInfo::new(false);
             }
 
-            // Mark expired messages for deletion. Marked messages will be deleted from the server
-            // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
-            // called right before `fetch_move_delete` because it is not well optimized and would
-            // otherwise slow down message fetching.
-            if let Err(err) = delete_expired_imap_messages(ctx)
-                .await
-                .context("delete_expired_imap_messages failed")
-            {
-                warn!(ctx, "{:#}", err);
-            }
-
-            // Scan additional folders only after finishing fetching the watched folder.
-            //
-            // On iOS the application has strictly limited time to work in background, so we may not
-            // be able to scan all folders before time is up if there are many of them.
-            if folder == Config::ConfiguredInboxFolder {
-                // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
-                match connection.scan_folders(ctx).await {
-                    Err(err) => {
-                        // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
-                        // but maybe just one folder can't be selected or something
-                        warn!(ctx, "{}", err);
-                    }
-                    Ok(true) => {
-                        // Fetch the watched folder again in case scanning other folder moved messages
-                        // there.
-                        //
-                        // In most cases this will select the watched folder and return because there are
-                        // no new messages. We want to select the watched folder anyway before going IDLE
-                        // there, so this does not take additional protocol round-trip.
-                        if let Err(err) = connection
-                            .fetch_move_delete(ctx, &watch_folder, false)
-                            .await
-                        {
-                            connection.trigger_reconnect(ctx).await;
-                            warn!(ctx, "{:#}", err);
-                            return InterruptInfo::new(false);
-                        }
-                    }
-                    Ok(false) => {}
-                }
-            }
-
-            // Synchronize Seen flags.
-            connection
-                .sync_seen_flags(ctx, &watch_folder)
-                .await
-                .context("sync_seen_flags")
-                .ok_or_log(ctx);
-
             connection.connectivity.set_connected(ctx).await;
 
             // idle
@@ -259,16 +222,17 @@ async fn simple_imap_loop(
     let ImapConnectionHandlers {
         mut connection,
         stop_receiver,
+        shutdown_sender,
     } = inbox_handlers;
 
     let ctx1 = ctx.clone();
 
     let fut = async move {
+        started
+            .send(())
+            .await
+            .expect("simple imap loop, missing started receive");
         let ctx = ctx1;
-        if let Err(err) = started.send(()).await {
-            warn!(&ctx, "simple imap loop, missing started receiver: {}", err);
-            return;
-        }
 
         loop {
             fetch_idle(&ctx, &mut connection, folder).await;
@@ -282,6 +246,10 @@ async fn simple_imap_loop(
         })
         .race(fut)
         .await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("simple imap loop, missing shutdown receiver");
 }
 
 async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnectionHandlers) {
@@ -291,46 +259,31 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
     let SmtpConnectionHandlers {
         mut connection,
         stop_receiver,
+        shutdown_sender,
         idle_interrupt_receiver,
     } = smtp_handlers;
 
     let ctx1 = ctx.clone();
     let fut = async move {
+        started
+            .send(())
+            .await
+            .expect("smtp loop, missing started receiver");
         let ctx = ctx1;
-        if let Err(err) = started.send(()).await {
-            warn!(&ctx, "smtp loop, missing started receiver: {}", err);
-            return;
-        }
 
-        let mut timeout = None;
         let mut interrupt_info = Default::default();
         loop {
-            let job = match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
-                Err(err) => {
-                    error!(ctx, "Failed loading job from the database: {:#}.", err);
-                    None
-                }
-                Ok(job) => job,
-            };
-
-            match job {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info)
+                .await
+                .ok()
+                .flatten()
+            {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
                     interrupt_info = Default::default();
                 }
                 None => {
-                    let res = send_smtp_messages(&ctx, &mut connection).await;
-                    if let Err(err) = &res {
-                        warn!(ctx, "send_smtp_messages failed: {:#}", err);
-                    }
-                    let success = res.unwrap_or(false);
-                    timeout = if success {
-                        None
-                    } else {
-                        Some(timeout.map_or(30, |timeout: u64| timeout.saturating_mul(3)))
-                    };
-
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
                     match &connection.last_send_error {
@@ -338,27 +291,7 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                         Some(err) => connection.connectivity.set_err(&ctx, err).await,
                     }
 
-                    // If send_smtp_messages() failed, we set a timeout for the fake-idle so that
-                    // sending is retried (at the latest) after the timeout. If sending fails
-                    // again, we increase the timeout exponentially, in order not to do lots of
-                    // unnecessary retries.
-                    if let Some(timeout) = timeout {
-                        info!(
-                            ctx,
-                            "smtp has messages to retry, planning to retry {} seconds later",
-                            timeout
-                        );
-                        let duration = std::time::Duration::from_secs(timeout);
-                        interrupt_info = async_std::future::timeout(duration, async {
-                            idle_interrupt_receiver.recv().await.unwrap_or_default()
-                        })
-                        .await
-                        .unwrap_or_default();
-                    } else {
-                        info!(ctx, "smtp has no messages to retry, waiting for interrupt");
-                        interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
-                    };
-
+                    interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
                     info!(ctx, "smtp fake idle - interrupted")
                 }
             }
@@ -372,15 +305,15 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
         })
         .race(fut)
         .await;
+    shutdown_sender
+        .send(())
+        .await
+        .expect("smtp loop, missing shutdown receiver");
 }
 
 impl Scheduler {
-    /// Start the scheduler, returns error if it is already running.
+    /// Start the scheduler, panics if it is already running.
     pub async fn start(&mut self, ctx: Context) -> Result<()> {
-        if self.is_running() {
-            bail!("scheduler is already started");
-        }
-
         let (mvbox, mvbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (sentbox, sentbox_handlers) = ImapConnectionState::new(&ctx).await?;
         let (smtp, smtp_handlers) = SmtpConnectionState::new();
@@ -392,8 +325,6 @@ impl Scheduler {
         let (sentbox_start_send, sentbox_start_recv) = channel::bounded(1);
         let mut sentbox_handle = None;
         let (smtp_start_send, smtp_start_recv) = channel::bounded(1);
-        let (ephemeral_interrupt_send, ephemeral_interrupt_recv) = channel::bounded(1);
-        let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1);
 
         let inbox_handle = {
             let ctx = ctx.clone();
@@ -402,7 +333,7 @@ impl Scheduler {
             }))
         };
 
-        if ctx.should_watch_mvbox().await? {
+        if ctx.get_config_bool(Config::MvboxMove).await? {
             let ctx = ctx.clone();
             mvbox_handle = Some(task::spawn(async move {
                 simple_imap_loop(
@@ -417,7 +348,7 @@ impl Scheduler {
             mvbox_start_send
                 .send(())
                 .await
-                .context("mvbox start send, missing receiver")?;
+                .expect("mvbox start send, missing receiver");
             mvbox_handlers
                 .connection
                 .connectivity
@@ -440,7 +371,7 @@ impl Scheduler {
             sentbox_start_send
                 .send(())
                 .await
-                .context("sentbox start send, missing receiver")?;
+                .expect("sentbox start send, missing receiver");
             sentbox_handlers
                 .connection
                 .connectivity
@@ -455,20 +386,6 @@ impl Scheduler {
             }))
         };
 
-        let ephemeral_handle = {
-            let ctx = ctx.clone();
-            Some(task::spawn(async move {
-                ephemeral::ephemeral_loop(&ctx, ephemeral_interrupt_recv).await;
-            }))
-        };
-
-        let location_handle = {
-            let ctx = ctx.clone();
-            Some(task::spawn(async move {
-                location::location_loop(&ctx, location_interrupt_recv).await;
-            }))
-        };
-
         *self = Scheduler::Running {
             inbox,
             mvbox,
@@ -478,10 +395,6 @@ impl Scheduler {
             mvbox_handle,
             sentbox_handle,
             smtp_handle,
-            ephemeral_handle,
-            ephemeral_interrupt_send,
-            location_handle,
-            location_interrupt_send,
         };
 
         // wait for all loops to be started
@@ -547,31 +460,11 @@ impl Scheduler {
         }
     }
 
-    async fn interrupt_ephemeral_task(&self) {
-        if let Scheduler::Running {
-            ref ephemeral_interrupt_send,
-            ..
-        } = self
-        {
-            ephemeral_interrupt_send.try_send(()).ok();
-        }
-    }
-
-    async fn interrupt_location(&self) {
-        if let Scheduler::Running {
-            ref location_interrupt_send,
-            ..
-        } = self
-        {
-            location_interrupt_send.try_send(()).ok();
-        }
-    }
-
-    /// Halt the scheduler.
-    pub(crate) async fn stop(&mut self) -> Result<()> {
+    /// Halts the scheduler, must be called first, and then `stop`.
+    pub(crate) async fn pre_stop(&self) -> StopToken {
         match self {
             Scheduler::Stopped => {
-                bail!("scheduler is already stopped");
+                panic!("WARN: already stopped");
             }
             Scheduler::Running {
                 inbox,
@@ -582,23 +475,39 @@ impl Scheduler {
                 sentbox_handle,
                 smtp,
                 smtp_handle,
-                ephemeral_handle,
-                location_handle,
                 ..
             } => {
                 if inbox_handle.is_some() {
-                    inbox.stop().await?;
+                    inbox.stop().await;
                 }
                 if mvbox_handle.is_some() {
-                    mvbox.stop().await?;
+                    mvbox.stop().await;
                 }
                 if sentbox_handle.is_some() {
-                    sentbox.stop().await?;
+                    sentbox.stop().await;
                 }
                 if smtp_handle.is_some() {
-                    smtp.stop().await?;
+                    smtp.stop().await;
                 }
 
+                StopToken
+            }
+        }
+    }
+
+    /// Halt the scheduler, must only be called after pre_stop.
+    pub(crate) async fn stop(&mut self, _t: StopToken) {
+        match self {
+            Scheduler::Stopped => {
+                panic!("WARN: already stopped");
+            }
+            Scheduler::Running {
+                inbox_handle,
+                mvbox_handle,
+                sentbox_handle,
+                smtp_handle,
+                ..
+            } => {
                 if let Some(handle) = inbox_handle.take() {
                     handle.await;
                 }
@@ -611,15 +520,8 @@ impl Scheduler {
                 if let Some(handle) = smtp_handle.take() {
                     handle.await;
                 }
-                if let Some(handle) = ephemeral_handle.take() {
-                    handle.cancel().await;
-                }
-                if let Some(handle) = location_handle.take() {
-                    handle.cancel().await;
-                }
 
                 *self = Scheduler::Stopped;
-                Ok(())
             }
         }
     }
@@ -633,6 +535,8 @@ impl Scheduler {
 /// Connection state logic shared between imap and smtp connections.
 #[derive(Debug)]
 struct ConnectionState {
+    /// Channel to notify that shutdown has completed.
+    shutdown_receiver: Receiver<()>,
     /// Channel to interrupt the whole connection.
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
@@ -643,13 +547,14 @@ struct ConnectionState {
 
 impl ConnectionState {
     /// Shutdown this connection completely.
-    async fn stop(&self) -> Result<()> {
+    async fn stop(&self) {
         // Trigger shutdown of the run loop.
         self.stop_sender
             .send(())
             .await
-            .context("failed to stop, missing receiver")?;
-        Ok(())
+            .expect("stop, missing receiver");
+        // Wait for a notification that the run loop has been shutdown.
+        self.shutdown_receiver.recv().await.ok();
     }
 
     async fn interrupt(&self, info: InterruptInfo) {
@@ -666,15 +571,18 @@ pub(crate) struct SmtpConnectionState {
 impl SmtpConnectionState {
     fn new() -> (Self, SmtpConnectionHandlers) {
         let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = SmtpConnectionHandlers {
             connection: Smtp::new(),
             stop_receiver,
+            shutdown_sender,
             idle_interrupt_receiver,
         };
 
         let state = ConnectionState {
+            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -691,15 +599,15 @@ impl SmtpConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) -> Result<()> {
-        self.state.stop().await?;
-        Ok(())
+    async fn stop(&self) {
+        self.state.stop().await;
     }
 }
 
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
     idle_interrupt_receiver: Receiver<InterruptInfo>,
 }
 
@@ -712,14 +620,17 @@ impl ImapConnectionState {
     /// Construct a new connection.
     async fn new(context: &Context) -> Result<(Self, ImapConnectionHandlers)> {
         let (stop_sender, stop_receiver) = channel::bounded(1);
+        let (shutdown_sender, shutdown_receiver) = channel::bounded(1);
         let (idle_interrupt_sender, idle_interrupt_receiver) = channel::bounded(1);
 
         let handlers = ImapConnectionHandlers {
             connection: Imap::new_configured(context, idle_interrupt_receiver).await?,
             stop_receiver,
+            shutdown_sender,
         };
 
         let state = ConnectionState {
+            shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
             connectivity: handlers.connection.connectivity.clone(),
@@ -736,9 +647,8 @@ impl ImapConnectionState {
     }
 
     /// Shutdown this connection completely.
-    async fn stop(&self) -> Result<()> {
-        self.state.stop().await?;
-        Ok(())
+    async fn stop(&self) {
+        self.state.stop().await;
     }
 }
 
@@ -746,6 +656,7 @@ impl ImapConnectionState {
 struct ImapConnectionHandlers {
     connection: Imap,
     stop_receiver: Receiver<()>,
+    shutdown_sender: Sender<()>,
 }
 
 #[derive(Default, Debug)]

src/scheduler/connectivity.rs

@@ -5,7 +5,6 @@ use async_std::sync::{Mutex, RwLockReadGuard};
 
 use crate::dc_tools::time;
 use crate::events::EventType;
-use crate::imap::scan_folders::get_watched_folder_configs;
 use crate::quota::{
     QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
 };
@@ -304,7 +303,7 @@ impl Context {
             <html>
             <head>
                 <meta charset="UTF-8" />
-                <meta name="viewport" content="initial-scale=1.0; user-scalable=no" />
+                <meta name="viewport" content="initial-scale=1.0" />
                 <style>
                     ul {
                         list-style-type: none;
@@ -363,14 +362,17 @@ impl Context {
                 [
                     (
                         Config::ConfiguredInboxFolder,
+                        None,
                         inbox.state.connectivity.clone(),
                     ),
                     (
                         Config::ConfiguredMvboxFolder,
+                        Some(Config::MvboxMove),
                         mvbox.state.connectivity.clone(),
                     ),
                     (
                         Config::ConfiguredSentboxFolder,
+                        Some(Config::SentboxWatch),
                         sentbox.state.connectivity.clone(),
                     ),
                 ],
@@ -389,12 +391,20 @@ impl Context {
         //                               - "Sent": Connected
         // =============================================================================================
 
-        let watched_folders = get_watched_folder_configs(self).await?;
         ret += &format!("<h3>{}</h3><ul>", stock_str::incoming_messages(self).await);
-        for (folder, state) in &folders_states {
-            let mut folder_added = false;
+        for (folder, watch, state) in &folders_states {
+            let w = if let Some(watch_config) = *watch {
+                self.get_config(watch_config)
+                    .await
+                    .ok_or_log(self)
+                    .flatten()
+                    == Some("1".to_string())
+            } else {
+                true
+            };
 
-            if watched_folders.contains(folder) {
+            let mut folder_added = false;
+            if w {
                 let f = self.get_config(*folder).await.ok_or_log(self).flatten();
 
                 if let Some(foldername) = f {
@@ -449,7 +459,13 @@ impl Context {
         //                                [======67%=====       ]
         // =============================================================================================
 
-        let domain = dc_tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain;
+        let domain = dc_tools::EmailAddress::new(
+            &self
+                .get_config(Config::ConfiguredAddr)
+                .await?
+                .unwrap_or_default(),
+        )?
+        .domain;
         ret += &format!(
             "<h3>{}</h3><ul>",
             stock_str::storage_on_domain(self, domain).await
@@ -538,7 +554,7 @@ impl Context {
                 self.schedule_quota_update().await?;
             }
         } else {
-            ret += &format!("<li>{}</li>", stock_str::not_connected(self).await);
+            ret += &format!("<li>{}</li>", stock_str::one_moment(self).await);
             self.schedule_quota_update().await?;
         }
         ret += "</ul>";

src/securejoin.rs

@@ -3,20 +3,21 @@
 use std::convert::TryFrom;
 
 use anyhow::{bail, Context as _, Error, Result};
+use async_std::sync::Mutex;
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{self, Chat, ChatId, ChatIdBlocked};
+use crate::chat::{self, is_contact_in_chat, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::Blocked;
-use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
+use crate::constants::{Blocked, Chattype, Viewtype, DC_CONTACT_ID_LAST_SPECIAL};
+use crate::contact::{Contact, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::e2ee::ensure_secret_key_exists;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
 use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus, ToSave};
@@ -24,16 +25,28 @@ use crate::qr::check_qr;
 use crate::stock_str;
 use crate::token;
 
-mod bob;
 mod bobstate;
 mod qrinvite;
 
 use crate::token::Namespace;
-use bobstate::BobState;
+use bobstate::{BobHandshakeStage, BobState, BobStateHandle};
 use qrinvite::QrInvite;
 
 pub const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.');
 
+macro_rules! joiner_progress {
+    ($context:tt, $contact_id:expr, $progress:expr) => {
+        assert!(
+            $progress >= 0 && $progress <= 1000,
+            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
+        );
+        $context.emit_event($crate::events::EventType::SecurejoinJoinerProgress {
+            contact_id: $contact_id,
+            progress: $progress,
+        });
+    };
+}
+
 macro_rules! inviter_progress {
     ($context:tt, $contact_id:expr, $progress:expr) => {
         assert!(
@@ -47,6 +60,100 @@ macro_rules! inviter_progress {
     };
 }
 
+/// State for setup-contact/secure-join protocol joiner's side, aka Bob's side.
+///
+/// The setup-contact protocol needs to carry state for both the inviter (Alice) and the
+/// joiner/invitee (Bob).  For Alice this state is minimal and in the `tokens` table in the
+/// database.  For Bob this state is only carried live on the [`Context`] in this struct.
+#[derive(Debug, Default)]
+pub(crate) struct Bob {
+    inner: Mutex<Option<BobState>>,
+}
+
+/// Return value for [`Bob::start_protocol`].
+///
+/// This indicates which protocol variant was started and provides the required information
+/// about it.
+enum StartedProtocolVariant {
+    /// The setup-contact protocol, to verify a contact.
+    SetupContact,
+    /// The secure-join protocol, to join a group.
+    SecureJoin {
+        group_id: String,
+        group_name: String,
+    },
+}
+
+impl Bob {
+    /// Starts the securejoin protocol with the QR `invite`.
+    ///
+    /// This will try to start the securejoin protocol for the given QR `invite`.  If it
+    /// succeeded the protocol state will be tracked in `self`.
+    ///
+    /// This function takes care of starting the "ongoing" mechanism if required and
+    /// handling errors while starting the protocol.
+    ///
+    /// # Returns
+    ///
+    /// If the started protocol is joining a group the returned struct contains information
+    /// about the group and ongoing process.
+    async fn start_protocol(
+        &self,
+        context: &Context,
+        invite: QrInvite,
+    ) -> Result<StartedProtocolVariant, JoinError> {
+        let mut guard = self.inner.lock().await;
+        if guard.is_some() {
+            warn!(context, "The new securejoin will replace the ongoing one.");
+            *guard = None;
+        }
+        let variant = match invite {
+            QrInvite::Group {
+                ref grpid,
+                ref name,
+                ..
+            } => StartedProtocolVariant::SecureJoin {
+                group_id: grpid.clone(),
+                group_name: name.clone(),
+            },
+            _ => StartedProtocolVariant::SetupContact,
+        };
+        match BobState::start_protocol(context, invite).await {
+            Ok((state, stage)) => {
+                if matches!(stage, BobHandshakeStage::RequestWithAuthSent) {
+                    joiner_progress!(context, state.invite().contact_id(), 400);
+                }
+                *guard = Some(state);
+                Ok(variant)
+            }
+            Err(err) => {
+                if let StartedProtocolVariant::SecureJoin { .. } = variant {
+                    context.free_ongoing().await;
+                }
+                Err(err)
+            }
+        }
+    }
+
+    /// Returns a handle to the [`BobState`] of the handshake.
+    ///
+    /// If there currently isn't a handshake running this will return `None`.  Otherwise
+    /// this will return a handle to the current [`BobState`].  This handle allows
+    /// processing an incoming message and allows terminating the handshake.
+    ///
+    /// The handle contains an exclusive lock, which is held until the handle is dropped.
+    /// This guarantees all state and state changes are correct and allows safely
+    /// terminating the handshake without worrying about concurrency.
+    async fn state(&self, context: &Context) -> Option<BobStateHandle<'_>> {
+        let guard = self.inner.lock().await;
+        let ret = BobStateHandle::from_guard(guard);
+        if ret.is_none() {
+            info!(context, "No active BobState found for securejoin handshake");
+        }
+        ret
+    }
+}
+
 /// Generates a Secure Join QR code.
 ///
 /// With `group` set to `None` this generates a setup-contact QR code, with `group` set to a
@@ -66,7 +173,19 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
         .is_none();
     let invitenumber = token::lookup_or_new(context, Namespace::InviteNumber, group).await;
     let auth = token::lookup_or_new(context, Namespace::Auth, group).await;
-    let self_addr = context.get_primary_self_addr().await?;
+    let self_addr = match context.get_config(Config::ConfiguredAddr).await {
+        Ok(Some(addr)) => addr,
+        Ok(None) => {
+            bail!("Not configured, cannot generate QR code.");
+        }
+        Err(err) => {
+            bail!(
+                "Unable to retrieve configuration, cannot generate QR code: {:?}",
+                err
+            );
+        }
+    };
+
     let self_name = context
         .get_config(Config::Displayname)
         .await?
@@ -87,12 +206,6 @@ pub async fn dc_get_securejoin_qr(context: &Context, group: Option<ChatId>) -> R
     let qr = if let Some(group) = group {
         // parameters used: a=g=x=i=s=
         let chat = Chat::load_from_db(context, group).await?;
-        if chat.grpid.is_empty() {
-            bail!(
-                "can't generate securejoin QR code for ad-hoc group {}",
-                group
-            );
-        }
         let group_name = chat.get_name();
         let group_name_urlencoded = utf8_percent_encode(group_name, NON_ALPHANUMERIC).to_string();
         if sync_token {
@@ -167,7 +280,7 @@ pub enum JoinError {
 pub async fn dc_join_securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
     securejoin(context, qr).await.map_err(|err| {
         warn!(context, "Fatal joiner error: {:#}", err);
-        // The user just scanned this QR code so has context on what failed.
+        // This is a modal operation, the user has context on what failed.
         error!(context, "QR process failed");
         err
     })
@@ -184,7 +297,47 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
 
     let invite = QrInvite::try_from(qr_scan)?;
 
-    bob::start_protocol(context, invite).await
+    match context.bob.start_protocol(context, invite.clone()).await? {
+        StartedProtocolVariant::SetupContact => {
+            // for a one-to-one-chat, the chat is already known, return the chat-id,
+            // the verification runs in background
+            let chat_id = ChatId::create_for_contact(context, invite.contact_id())
+                .await
+                .map_err(JoinError::UnknownContact)?;
+            Ok(chat_id)
+        }
+        StartedProtocolVariant::SecureJoin {
+            group_id,
+            group_name,
+        } => {
+            // for a group-join, also create the chat soon and let the verification run in background.
+            // however, the group will become usable only when the protocol has finished.
+            let contact_id = invite.contact_id();
+            let chat_id = if let Some((chat_id, _protected, _blocked)) =
+                chat::get_chat_id_by_grpid(context, &group_id).await?
+            {
+                chat_id.unblock(context).await?;
+                chat_id
+            } else {
+                ChatId::create_multiuser_record(
+                    context,
+                    Chattype::Group,
+                    &group_id,
+                    &group_name,
+                    Blocked::Not,
+                    ProtectionStatus::Unprotected, // protection is added later as needed
+                    None,
+                )
+                .await?
+            };
+            if !is_contact_in_chat(context, chat_id, contact_id).await? {
+                chat::add_to_chat_contacts_table(context, chat_id, contact_id).await?;
+            }
+            let msg = stock_str::secure_join_started(context, contact_id).await;
+            chat::add_info_msg(context, chat_id, &msg, time()).await?;
+            Ok(chat_id)
+        }
+    }
 }
 
 /// Error when failing to send a protocol handshake message.
@@ -199,7 +352,7 @@ pub struct SendMsgError(#[from] anyhow::Error);
 /// Bob's handshake messages are sent in `BobState::send_handshake_message()`.
 async fn send_alice_handshake_msg(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     step: &str,
     fingerprint: Option<Fingerprint>,
 ) -> Result<(), SendMsgError> {
@@ -227,7 +380,7 @@ async fn send_alice_handshake_msg(
 }
 
 /// Get an unblocked chat that can be used for info messages.
-async fn info_chat_id(context: &Context, contact_id: ContactId) -> Result<ChatId> {
+async fn info_chat_id(context: &Context, contact_id: u32) -> Result<ChatId> {
     let chat_id_blocked = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Not).await?;
     Ok(chat_id_blocked.id)
 }
@@ -235,7 +388,7 @@ async fn info_chat_id(context: &Context, contact_id: ContactId) -> Result<ChatId
 async fn fingerprint_equals_sender(
     context: &Context,
     fingerprint: &Fingerprint,
-    contact_id: ContactId,
+    contact_id: u32,
 ) -> Result<bool, Error> {
     let contact = Contact::load_from_db(context, contact_id).await?;
     let peerstate = match Peerstate::from_addr(context, contact.get_addr()).await {
@@ -289,8 +442,9 @@ pub(crate) enum HandshakeMessage {
 
 /// Handle incoming secure-join handshake.
 ///
-/// This function will update the securejoin state in the database as the protocol
-/// progresses.
+/// This function will update the securejoin state in [`InnerContext::bob`] and also
+/// terminate the ongoing process using [`Context::stop_ongoing`] as required by the
+/// protocol.
 ///
 /// A message which results in [`Err`] will be hidden from the user but not deleted, it may
 /// be a valid message for something else we are not aware off.  E.g. it could be part of a
@@ -298,13 +452,15 @@ pub(crate) enum HandshakeMessage {
 ///
 /// When `handle_securejoin_handshake()` is called, the message is not yet filed in the
 /// database; this is done by `receive_imf()` later on as needed.
+///
+/// [`InnerContext::bob`]: crate::context::InnerContext::bob
 #[allow(clippy::indexing_slicing)]
 pub(crate) async fn handle_securejoin_handshake(
     context: &Context,
     mime_message: &MimeMessage,
-    contact_id: ContactId,
+    contact_id: u32,
 ) -> Result<HandshakeMessage> {
-    if contact_id.is_special() {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -365,7 +521,38 @@ pub(crate) async fn handle_securejoin_handshake(
             ====             Bob - the joiner's side             =====
             ====   Step 4 in "Setup verified contact" protocol   =====
             ========================================================*/
-            bob::handle_auth_required(context, mime_message).await
+            match context.bob.state(context).await {
+                Some(mut bobstate) => match bobstate.handle_message(context, mime_message).await {
+                    Some(BobHandshakeStage::Terminated(why)) => {
+                        could_not_establish_secure_connection(
+                            context,
+                            contact_id,
+                            bobstate.chat_id(context).await?,
+                            why,
+                        )
+                        .await?;
+                        Ok(HandshakeMessage::Done)
+                    }
+                    Some(_stage) => {
+                        if join_vg {
+                            // the message reads "Alice replied, waiting for being added to the group…";
+                            // show it only on secure-join and not on setup-contact therefore.
+                            let msg = stock_str::secure_join_replies(context, contact_id).await;
+                            chat::add_info_msg(
+                                context,
+                                bobstate.chat_id(context).await?,
+                                &msg,
+                                time(),
+                            )
+                            .await?;
+                        }
+                        joiner_progress!(context, bobstate.invite().contact_id(), 400);
+                        Ok(HandshakeMessage::Done)
+                    }
+                    None => Ok(HandshakeMessage::Ignore),
+                },
+                None => Ok(HandshakeMessage::Ignore),
+            }
         }
         "vg-request-with-auth" | "vc-request-with-auth" => {
             /*==========================================================
@@ -496,14 +683,44 @@ pub(crate) async fn handle_securejoin_handshake(
             ====             Bob - the joiner's side             ====
             ====   Step 7 in "Setup verified contact" protocol   ====
             =======================================================*/
-            match BobState::from_db(&context.sql).await? {
-                Some(bobstate) => {
-                    bob::handle_contact_confirm(context, bobstate, mime_message).await
-                }
-                None => match join_vg {
-                    true => Ok(HandshakeMessage::Propagate),
-                    false => Ok(HandshakeMessage::Ignore),
+            info!(context, "matched vc-contact-confirm step");
+            let retval = if join_vg {
+                HandshakeMessage::Propagate
+            } else {
+                HandshakeMessage::Ignore
+            };
+            match context.bob.state(context).await {
+                Some(mut bobstate) => match bobstate.handle_message(context, mime_message).await {
+                    Some(BobHandshakeStage::Terminated(why)) => {
+                        could_not_establish_secure_connection(
+                            context,
+                            contact_id,
+                            bobstate.chat_id(context).await?,
+                            why,
+                        )
+                        .await?;
+                        Ok(HandshakeMessage::Done)
+                    }
+                    Some(BobHandshakeStage::Completed) => {
+                        // Can only be BobHandshakeStage::Completed
+                        secure_connection_established(
+                            context,
+                            contact_id,
+                            bobstate.chat_id(context).await?,
+                        )
+                        .await?;
+                        Ok(retval)
+                    }
+                    Some(_) => {
+                        warn!(
+                            context,
+                            "Impossible state returned from handling handshake message"
+                        );
+                        Ok(retval)
+                    }
+                    None => Ok(retval),
                 },
+                None => Ok(retval),
             }
         }
         "vg-member-added-received" | "vc-contact-confirm-received" => {
@@ -565,9 +782,9 @@ pub(crate) async fn handle_securejoin_handshake(
 pub(crate) async fn observe_securejoin_on_other_device(
     context: &Context,
     mime_message: &MimeMessage,
-    contact_id: ContactId,
+    contact_id: u32,
 ) -> Result<HandshakeMessage> {
-    if contact_id.is_special() {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
@@ -630,11 +847,11 @@ pub(crate) async fn observe_securejoin_on_other_device(
 
 async fn secure_connection_established(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     chat_id: ChatId,
 ) -> Result<(), Error> {
     let contact = Contact::get_by_id(context, contact_id).await?;
-    let msg = stock_str::contact_verified(context, &contact).await;
+    let msg = stock_str::contact_verified(context, contact.get_name_n_addr()).await;
     chat::add_info_msg(context, chat_id, &msg, time()).await?;
     context.emit_event(EventType::ChatModified(chat_id));
     Ok(())
@@ -642,12 +859,20 @@ async fn secure_connection_established(
 
 async fn could_not_establish_secure_connection(
     context: &Context,
-    contact_id: ContactId,
+    contact_id: u32,
     chat_id: ChatId,
     details: &str,
 ) -> Result<(), Error> {
-    let contact = Contact::get_by_id(context, contact_id).await?;
-    let msg = stock_str::contact_not_verified(context, &contact).await;
+    let contact = Contact::get_by_id(context, contact_id).await;
+    let msg = stock_str::contact_not_verified(
+        context,
+        if let Ok(ref contact) = contact {
+            contact.get_addr()
+        } else {
+            "?"
+        },
+    )
+    .await;
     chat::add_info_msg(context, chat_id, &msg, time()).await?;
     warn!(
         context,
@@ -716,36 +941,31 @@ mod tests {
     use crate::chat::ProtectionStatus;
     use crate::chatlist::Chatlist;
     use crate::constants::Chattype;
-    use crate::dc_receive_imf::dc_receive_imf;
     use crate::peerstate::Peerstate;
-    use crate::test_utils::{TestContext, TestContextManager};
+    use crate::test_utils::{LogSink, TestContext};
 
     #[async_std::test]
-    async fn test_setup_contact() {
-        let mut tcm = TestContextManager::new().await;
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
-        assert_eq!(
-            Chatlist::try_load(&alice, 0, None, None)
-                .await
-                .unwrap()
-                .len(),
-            0
-        );
-        assert_eq!(
-            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
-            0
-        );
+    async fn test_setup_contact() -> Result<()> {
+        let (log_tx, _log_sink) = LogSink::create();
+        let alice = TestContext::builder()
+            .configure_alice()
+            .with_log_sink(log_tx.clone())
+            .build()
+            .await;
+        let bob = TestContext::builder()
+            .configure_bob()
+            .with_log_sink(log_tx)
+            .build()
+            .await;
+        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
+        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 0);
 
         // Step 1: Generate QR-code, ChatId(0) indicates setup-contact
-        let qr = dc_get_securejoin_qr(&alice.ctx, None).await.unwrap();
+        let qr = dc_get_securejoin_qr(&alice.ctx, None).await?;
 
         // Step 2: Bob scans QR-code, sends vc-request
-        dc_join_securejoin(&bob.ctx, &qr).await.unwrap();
-        assert_eq!(
-            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
-            1
-        );
+        dc_join_securejoin(&bob.ctx, &qr).await?;
+        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 1);
 
         let sent = bob.pop_sent_msg().await;
         assert!(!bob.ctx.has_ongoing().await);
@@ -757,13 +977,7 @@ mod tests {
 
         // Step 3: Alice receives vc-request, sends vc-auth-required
         alice.recv_msg(&sent).await;
-        assert_eq!(
-            Chatlist::try_load(&alice, 0, None, None)
-                .await
-                .unwrap()
-                .len(),
-            1
-        );
+        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);
 
         let sent = alice.pop_sent_msg().await;
         let msg = bob.parse_msg(&sent).await;
@@ -825,30 +1039,21 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(
-            contact_bob.is_verified(&alice.ctx).await.unwrap(),
+            contact_bob.is_verified(&alice.ctx).await?,
             VerifiedStatus::Unverified
         );
 
         // Step 5+6: Alice receives vc-request-with-auth, sends vc-contact-confirm
         alice.recv_msg(&sent).await;
         assert_eq!(
-            contact_bob.is_verified(&alice.ctx).await.unwrap(),
+            contact_bob.is_verified(&alice.ctx).await?,
             VerifiedStatus::BidirectVerified
         );
 
         // exactly one one-to-one chat should be visible for both now
         // (check this before calling alice.create_chat() explicitly below)
-        assert_eq!(
-            Chatlist::try_load(&alice, 0, None, None)
-                .await
-                .unwrap()
-                .len(),
-            1
-        );
-        assert_eq!(
-            Chatlist::try_load(&bob, 0, None, None).await.unwrap().len(),
-            1
-        );
+        assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 1);
+        assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 1);
 
         // Check Alice got the verified message in her 1:1 chat.
         {
@@ -888,14 +1093,14 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(
-            contact_bob.is_verified(&bob.ctx).await.unwrap(),
+            contact_bob.is_verified(&bob.ctx).await?,
             VerifiedStatus::Unverified
         );
 
         // Step 7: Bob receives vc-contact-confirm, sends vc-contact-confirm-received
         bob.recv_msg(&sent).await;
         assert_eq!(
-            contact_alice.is_verified(&bob.ctx).await.unwrap(),
+            contact_alice.is_verified(&bob.ctx).await?,
             VerifiedStatus::BidirectVerified
         );
 
@@ -926,6 +1131,7 @@ mod tests {
             msg.get_header(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm-received"
         );
+        Ok(())
     }
 
     #[async_std::test]
@@ -937,9 +1143,17 @@ mod tests {
 
     #[async_std::test]
     async fn test_setup_contact_bob_knows_alice() -> Result<()> {
-        let mut tcm = TestContextManager::new().await;
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
+        let (log_tx, _log_sink) = LogSink::create();
+        let alice = TestContext::builder()
+            .configure_alice()
+            .with_log_sink(log_tx.clone())
+            .build()
+            .await;
+        let bob = TestContext::builder()
+            .configure_bob()
+            .with_log_sink(log_tx)
+            .build()
+            .await;
 
         // Ensure Bob knows Alice_FP
         let alice_pubkey = SignedPublicKey::load_self(&alice.ctx).await?;
@@ -1062,9 +1276,17 @@ mod tests {
 
     #[async_std::test]
     async fn test_setup_contact_concurrent_calls() -> Result<()> {
-        let mut tcm = TestContextManager::new().await;
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
+        let (log_tx, _log_sink) = LogSink::create();
+        let alice = TestContext::builder()
+            .configure_alice()
+            .with_log_sink(log_tx.clone())
+            .build()
+            .await;
+        let bob = TestContext::builder()
+            .configure_bob()
+            .with_log_sink(log_tx)
+            .build()
+            .await;
 
         // do a scan that is not working as claire is never responding
         let qr_stale = "OPENPGP4FPR:1234567890123456789012345678901234567890#a=claire%40foo.de&n=&i=12345678901&s=23456789012";
@@ -1093,19 +1315,25 @@ mod tests {
 
     #[async_std::test]
     async fn test_secure_join() -> Result<()> {
-        let mut tcm = TestContextManager::new().await;
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
-
-        // We start with empty chatlists.
+        let (log_tx, _log_sink) = LogSink::create();
+        let alice = TestContext::builder()
+            .configure_alice()
+            .with_log_sink(log_tx.clone())
+            .build()
+            .await;
+        let bob = TestContext::builder()
+            .configure_bob()
+            .with_log_sink(log_tx)
+            .build()
+            .await;
         assert_eq!(Chatlist::try_load(&alice, 0, None, None).await?.len(), 0);
         assert_eq!(Chatlist::try_load(&bob, 0, None, None).await?.len(), 0);
 
-        let alice_chatid =
+        let chatid =
             chat::create_group_chat(&alice.ctx, ProtectionStatus::Protected, "the chat").await?;
 
         // Step 1: Generate QR-code, secure-join implied by chatid
-        let qr = dc_get_securejoin_qr(&alice.ctx, Some(alice_chatid))
+        let qr = dc_get_securejoin_qr(&alice.ctx, Some(chatid))
             .await
             .unwrap();
 
@@ -1196,35 +1424,6 @@ mod tests {
             "vg-member-added"
         );
 
-        {
-            // Now Alice's chat with Bob should still be hidden, the verified message should
-            // appear in the group chat.
-
-            let chat = alice
-                .get_chat(&bob)
-                .await
-                .expect("Alice has no 1:1 chat with bob");
-            assert_eq!(
-                chat.blocked,
-                Blocked::Yes,
-                "Alice's 1:1 chat with Bob is not hidden"
-            );
-            let msg_id = chat::get_chat_msgs(&alice.ctx, alice_chatid, 0x1, None)
-                .await
-                .unwrap()
-                .into_iter()
-                .filter_map(|item| match item {
-                    chat::ChatItem::Message { msg_id } => Some(msg_id),
-                    _ => None,
-                })
-                .min()
-                .expect("No messages in Alice's group chat");
-            let msg = Message::load_from_db(&alice.ctx, msg_id).await.unwrap();
-            assert!(msg.is_info());
-            let text = msg.get_text().unwrap();
-            assert!(text.contains("bob@example.net verified"));
-        }
-
         // Bob should not yet have Alice verified
         let contact_alice_id =
             Contact::lookup_id_by_addr(&bob.ctx, "alice@example.org", Origin::Unknown)
@@ -1239,53 +1438,10 @@ mod tests {
 
         // Step 7: Bob receives vg-member-added, sends vg-member-added-received
         bob.recv_msg(&sent).await;
-        {
-            // Bob has Alice verified, message shows up in the group chat.
-            assert_eq!(
-                contact_alice.is_verified(&bob.ctx).await?,
-                VerifiedStatus::BidirectVerified
-            );
-            let chat = bob
-                .get_chat(&alice)
-                .await
-                .expect("Bob has no 1:1 chat with Alice");
-            assert_eq!(
-                chat.blocked,
-                Blocked::Yes,
-                "Bob's 1:1 chat with Alice is not hidden"
-            );
-            for item in chat::get_chat_msgs(&bob.ctx, bob_chatid, 0x1, None)
-                .await
-                .unwrap()
-            {
-                if let chat::ChatItem::Message { msg_id } = item {
-                    let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap();
-                    let text = msg.get_text().unwrap();
-                    println!("msg {} text: {}", msg_id, text);
-                }
-            }
-            let mut msg_iter = chat::get_chat_msgs(&bob.ctx, bob_chatid, 0x1, None)
-                .await
-                .unwrap()
-                .into_iter();
-            loop {
-                match msg_iter.next() {
-                    Some(chat::ChatItem::Message { msg_id }) => {
-                        let msg = Message::load_from_db(&bob.ctx, msg_id).await.unwrap();
-                        let text = msg.get_text().unwrap();
-                        match text.contains("alice@example.org verified") {
-                            true => {
-                                assert!(msg.is_info());
-                                break;
-                            }
-                            false => continue,
-                        }
-                    }
-                    Some(_) => continue,
-                    None => panic!("Verified message not found in Bob's group chat"),
-                }
-            }
-        }
+        assert_eq!(
+            contact_alice.is_verified(&bob.ctx).await?,
+            VerifiedStatus::BidirectVerified
+        );
 
         let sent = bob.pop_sent_msg().await;
         let msg = alice.parse_msg(&sent).await;
@@ -1315,25 +1471,4 @@ mod tests {
 
         Ok(())
     }
-
-    #[async_std::test]
-    async fn test_adhoc_group_no_qr() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        alice.set_config(Config::ShowEmails, Some("2")).await?;
-
-        let mime = br#"Subject: First thread
-Message-ID: first@example.org
-To: Alice <alice@example.org>, Bob <bob@example.net>
-From: Claire <claire@example.org>
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-First thread."#;
-
-        dc_receive_imf(&alice, mime, false).await?;
-        let msg = alice.get_last_msg().await;
-        let chat_id = msg.chat_id;
-
-        assert!(dc_get_securejoin_qr(&alice, Some(chat_id)).await.is_err());
-        Ok(())
-    }
 }

src/securejoin/bob.rs

@@ -1,257 +0,0 @@
-//! Bob's side of SecureJoin handling.
-//!
-//! This are some helper functions around [`BobState`] which augment the state changes with
-//! the required user interactions.
-
-use anyhow::Result;
-
-use crate::chat::{is_contact_in_chat, ChatId, ProtectionStatus};
-use crate::constants::{Blocked, Chattype};
-use crate::contact::Contact;
-use crate::context::Context;
-use crate::dc_tools::time;
-use crate::events::EventType;
-use crate::mimeparser::MimeMessage;
-use crate::{chat, stock_str};
-
-use super::bobstate::{BobHandshakeStage, BobState};
-use super::qrinvite::QrInvite;
-use super::{HandshakeMessage, JoinError};
-
-/// Starts the securejoin protocol with the QR `invite`.
-///
-/// This will try to start the securejoin protocol for the given QR `invite`.  If it
-/// succeeded the protocol state will be tracked in `self`.
-///
-/// This function takes care of handling multiple concurrent joins and handling errors while
-/// starting the protocol.
-///
-/// # Returns
-///
-/// The [`ChatId`] of the created chat is returned, for a SetupContact QR this is the 1:1
-/// chat with Alice, for a SecureJoin QR this is the group chat.
-pub(super) async fn start_protocol(
-    context: &Context,
-    invite: QrInvite,
-) -> Result<ChatId, JoinError> {
-    // A 1:1 chat is needed to send messages to Alice.  When joining a group this chat is
-    // hidden, if a user starts sending messages in it it will be unhidden in
-    // dc_receive_imf.
-    let hidden = match invite {
-        QrInvite::Contact { .. } => Blocked::Not,
-        QrInvite::Group { .. } => Blocked::Yes,
-    };
-    let chat_id = ChatId::create_for_contact_with_blocked(context, invite.contact_id(), hidden)
-        .await
-        .map_err(JoinError::UnknownContact)?;
-
-    // Now start the protocol and initialise the state
-    let (state, stage, aborted_states) =
-        BobState::start_protocol(context, invite.clone(), chat_id).await?;
-    for state in aborted_states {
-        error!(context, "Aborting previously unfinished QR Join process.");
-        state.notify_aborted(context, "new QR scanned").await?;
-        state.emit_progress(context, JoinerProgress::Error);
-    }
-    if matches!(stage, BobHandshakeStage::RequestWithAuthSent) {
-        state.emit_progress(context, JoinerProgress::RequestWithAuthSent);
-    }
-    match invite {
-        QrInvite::Group { .. } => {
-            // For a secure-join we need to create the group and add the contact.  The group will
-            // only become usable once the protocol is finished.
-            // TODO: how does this group become usable?
-            let group_chat_id = state.joining_chat_id(context).await?;
-            if !is_contact_in_chat(context, group_chat_id, invite.contact_id()).await? {
-                chat::add_to_chat_contacts_table(context, group_chat_id, invite.contact_id())
-                    .await?;
-            }
-            let msg = stock_str::secure_join_started(context, invite.contact_id()).await;
-            chat::add_info_msg(context, group_chat_id, &msg, time()).await?;
-            Ok(group_chat_id)
-        }
-        QrInvite::Contact { .. } => {
-            // For setup-contact the BobState already ensured the 1:1 chat exists because it
-            // uses it to send the handshake messages.
-            Ok(state.alice_chat())
-        }
-    }
-}
-
-/// Handles `vc-auth-required` and `vg-auth-required` handshake messages.
-///
-/// # Bob - the joiner's side
-/// ## Step 4 in the "Setup Contact protocol"
-pub(super) async fn handle_auth_required(
-    context: &Context,
-    message: &MimeMessage,
-) -> Result<HandshakeMessage> {
-    match BobState::from_db(&context.sql).await? {
-        Some(mut bobstate) => match bobstate.handle_message(context, message).await? {
-            Some(BobHandshakeStage::Terminated(why)) => {
-                bobstate.notify_aborted(context, why).await?;
-                Ok(HandshakeMessage::Done)
-            }
-            Some(_stage) => {
-                if bobstate.is_join_group() {
-                    // The message reads "Alice replied, waiting to be added to the group…",
-                    // so only show it on secure-join and not on setup-contact.
-                    let contact_id = bobstate.invite().contact_id();
-                    let msg = stock_str::secure_join_replies(context, contact_id).await;
-                    let chat_id = bobstate.joining_chat_id(context).await?;
-                    chat::add_info_msg(context, chat_id, &msg, time()).await?;
-                }
-                bobstate.emit_progress(context, JoinerProgress::RequestWithAuthSent);
-                Ok(HandshakeMessage::Done)
-            }
-            None => Ok(HandshakeMessage::Ignore),
-        },
-        None => Ok(HandshakeMessage::Ignore),
-    }
-}
-
-/// Handles `vc-contact-confirm` and `vg-member-added` handshake messages.
-///
-/// # Bob - the joiner's side
-/// ## Step 4 in the "Setup Contact protocol"
-pub(super) async fn handle_contact_confirm(
-    context: &Context,
-    mut bobstate: BobState,
-    message: &MimeMessage,
-) -> Result<HandshakeMessage> {
-    let retval = if bobstate.is_join_group() {
-        HandshakeMessage::Propagate
-    } else {
-        HandshakeMessage::Ignore
-    };
-    match bobstate.handle_message(context, message).await? {
-        Some(BobHandshakeStage::Terminated(why)) => {
-            bobstate.notify_aborted(context, why).await?;
-            Ok(HandshakeMessage::Done)
-        }
-        Some(BobHandshakeStage::Completed) => {
-            // Note this goes to the 1:1 chat, as when joining a group we implicitly also
-            // verify both contacts (this could be a bug/security issue, see
-            // e.g. https://github.com/deltachat/deltachat-core-rust/issues/1177).
-            bobstate.notify_peer_verified(context).await?;
-            Ok(retval)
-        }
-        Some(_) => {
-            warn!(
-                context,
-                "Impossible state returned from handling handshake message"
-            );
-            Ok(retval)
-        }
-        None => Ok(retval),
-    }
-}
-
-/// Private implementations for user interactions about this [`BobState`].
-impl BobState {
-    fn is_join_group(&self) -> bool {
-        match self.invite() {
-            QrInvite::Contact { .. } => false,
-            QrInvite::Group { .. } => true,
-        }
-    }
-
-    fn emit_progress(&self, context: &Context, progress: JoinerProgress) {
-        let contact_id = self.invite().contact_id();
-        context.emit_event(EventType::SecurejoinJoinerProgress {
-            contact_id,
-            progress: progress.into(),
-        });
-    }
-
-    /// Returns the [`ChatId`] of the chat being joined.
-    ///
-    /// This is the chat in which you want to notify the user as well.
-    ///
-    /// When joining a group this is the [`ChatId`] of the group chat, when verifying a
-    /// contact this is the [`ChatId`] of the 1:1 chat.  The 1:1 chat is assumed to exist
-    /// because a [`BobState`] can not exist without, the group chat will be created if it
-    /// does not yet exist.
-    async fn joining_chat_id(&self, context: &Context) -> Result<ChatId> {
-        match self.invite() {
-            QrInvite::Contact { .. } => Ok(self.alice_chat()),
-            QrInvite::Group {
-                ref grpid,
-                ref name,
-                ..
-            } => {
-                let group_chat_id = match chat::get_chat_id_by_grpid(context, grpid).await? {
-                    Some((chat_id, _protected, _blocked)) => {
-                        chat_id.unblock(context).await?;
-                        chat_id
-                    }
-                    None => {
-                        ChatId::create_multiuser_record(
-                            context,
-                            Chattype::Group,
-                            grpid,
-                            name,
-                            Blocked::Not,
-                            ProtectionStatus::Unprotected, // protection is added later as needed
-                            None,
-                        )
-                        .await?
-                    }
-                };
-                Ok(group_chat_id)
-            }
-        }
-    }
-
-    /// Notifies the user that the SecureJoin was aborted.
-    ///
-    /// This creates an info message in the chat being joined.
-    async fn notify_aborted(&self, context: &Context, why: &str) -> Result<()> {
-        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let msg = stock_str::contact_not_verified(context, &contact).await;
-        let chat_id = self.joining_chat_id(context).await?;
-        chat::add_info_msg(context, chat_id, &msg, time()).await?;
-        warn!(
-            context,
-            "StockMessage::ContactNotVerified posted to joining chat ({})", why
-        );
-        Ok(())
-    }
-
-    /// Notifies the user that the SecureJoin peer is verified.
-    ///
-    /// This creates an info message in the chat being joined.
-    async fn notify_peer_verified(&self, context: &Context) -> Result<()> {
-        let contact = Contact::get_by_id(context, self.invite().contact_id()).await?;
-        let msg = stock_str::contact_verified(context, &contact).await;
-        let chat_id = self.joining_chat_id(context).await?;
-        chat::add_info_msg(context, chat_id, &msg, time()).await?;
-        context.emit_event(EventType::ChatModified(chat_id));
-        Ok(())
-    }
-}
-
-/// Progress updates for [`EventType::SecurejoinJoinerProgress`].
-///
-/// This has an `From<JoinerProgress> for usize` impl yielding numbers between 0 and a 1000
-/// which can be shown as a progress bar.
-enum JoinerProgress {
-    /// An error occurred.
-    Error,
-    /// vg-vc-request-with-auth sent.
-    ///
-    /// Typically shows as "alice@addr verified, introducing myself."
-    RequestWithAuthSent,
-    // /// Completed securejoin.
-    // Succeeded,
-}
-
-impl From<JoinerProgress> for usize {
-    fn from(progress: JoinerProgress) -> Self {
-        match progress {
-            JoinerProgress::Error => 0,
-            JoinerProgress::RequestWithAuthSent => 400,
-            // JoinerProgress::Succeeded => 1000,
-        }
-    }
-}

src/securejoin/bobstate.rs

@@ -5,21 +5,22 @@
 //! provides all the information to its driver so it can perform the correct interactions.
 //!
 //! The [`BobState`] is only directly used to initially create it when starting the
-//! protocol.
+//! protocol.  Afterwards it must be stored in a mutex and the [`BobStateHandle`] should be
+//! used to work with the state.
 
-use anyhow::{Error, Result};
-use rusqlite::Connection;
+use anyhow::{bail, Error, Result};
+use async_std::sync::MutexGuard;
 
 use crate::chat::{self, ChatId};
+use crate::constants::{Blocked, Viewtype};
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
 use crate::key::{DcKey, SignedPublicKey};
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
-use crate::sql::Sql;
 
 use super::qrinvite::QrInvite;
 use super::{
@@ -45,14 +46,123 @@ pub enum BobHandshakeStage {
     Terminated(&'static str),
 }
 
-/// The securejoin state kept while Bob is joining.
+/// A handle to work with the [`BobState`] of Bob's securejoin protocol.
 ///
-/// This is stored in the database and loaded from there using [`BobState::from_db`].  To
-/// create a new one use [`BobState::start_protocol`].
+/// This handle can only be created for when an underlying [`BobState`] exists.  It keeps
+/// open a lock which guarantees unique access to the state and this struct must be dropped
+/// to return the lock.
+pub struct BobStateHandle<'a> {
+    guard: MutexGuard<'a, Option<BobState>>,
+    bobstate: BobState,
+    clear_state_on_drop: bool,
+}
+
+impl<'a> BobStateHandle<'a> {
+    /// Creates a new instance, upholding the guarantee that [`BobState`] must exist.
+    pub fn from_guard(mut guard: MutexGuard<'a, Option<BobState>>) -> Option<Self> {
+        guard.take().map(|bobstate| Self {
+            guard,
+            bobstate,
+            clear_state_on_drop: false,
+        })
+    }
+
+    /// Returns the [`ChatId`] of the group chat to join or the 1:1 chat with Alice.
+    pub async fn chat_id(&self, context: &Context) -> Result<ChatId> {
+        match self.bobstate.invite {
+            QrInvite::Group { ref grpid, .. } => {
+                if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, grpid).await? {
+                    Ok(chat_id)
+                } else {
+                    bail!("chat not found")
+                }
+            }
+            QrInvite::Contact { .. } => Ok(self.bobstate.chat_id),
+        }
+    }
+
+    /// Returns a reference to the [`QrInvite`] of the joiner process.
+    pub fn invite(&self) -> &QrInvite {
+        &self.bobstate.invite
+    }
+
+    /// Handles the given message for the securejoin handshake for Bob.
+    ///
+    /// This proxies to [`BobState::handle_message`] and makes sure to clear the state when
+    /// the protocol state is terminal.  It returns `Some` if the message successfully
+    /// advanced the state of the protocol state machine, `None` otherwise.
+    pub async fn handle_message(
+        &mut self,
+        context: &Context,
+        mime_message: &MimeMessage,
+    ) -> Option<BobHandshakeStage> {
+        info!(context, "Handling securejoin message for BobStateHandle");
+        match self.bobstate.handle_message(context, mime_message).await {
+            Ok(Some(stage)) => {
+                if matches!(
+                    stage,
+                    BobHandshakeStage::Completed | BobHandshakeStage::Terminated(_)
+                ) {
+                    self.finish_protocol(context).await;
+                }
+                Some(stage)
+            }
+            Ok(None) => None,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Error handling handshake message, aborting handshake: {}", err
+                );
+                self.finish_protocol(context).await;
+                None
+            }
+        }
+    }
+
+    /// Marks the bob handshake as finished.
+    ///
+    /// This will clear the state on [`InnerContext::bob`] once this handle is dropped,
+    /// allowing a new handshake to be started from [`Bob`].
+    ///
+    /// Note that the state is only cleared on Drop since otherwise the invariant that the
+    /// state is always consistent is violated.  However the "ongoing" process is released
+    /// here a little bit earlier as this requires access to the Context, which we do not
+    /// have on Drop (Drop can not run asynchronous code).  Stopping the "ongoing" process
+    /// will release [`securejoin`](super::securejoin) which in turn will finally free the
+    /// ongoing process using [`Context::free_ongoing`].
+    ///
+    /// [`InnerContext::bob`]: crate::context::InnerContext::bob
+    /// [`Bob`]: super::Bob
+    async fn finish_protocol(&mut self, context: &Context) {
+        info!(context, "Finishing securejoin handshake protocol for Bob");
+        self.clear_state_on_drop = true;
+        if let QrInvite::Group { .. } = self.bobstate.invite {
+            context.stop_ongoing().await;
+        }
+    }
+}
+
+impl<'a> Drop for BobStateHandle<'a> {
+    fn drop(&mut self) {
+        if self.clear_state_on_drop {
+            // The Option should already be empty because we take it out in the ctor,
+            // however the typesystem doesn't guarantee this so do it again anyway.
+            self.guard.take();
+        } else {
+            // Make sure to put back the BobState into the Option of the Mutex, it was taken
+            // out by the constructor.
+            self.guard.replace(self.bobstate.clone());
+        }
+    }
+}
+
+/// The securejoin state kept in-memory while Bob is joining.
 ///
-/// This purposefully has nothing optional, the state is always fully valid.  However once a
-/// terminal state is reached in [`BobState::next`] the entry in the database will already
-/// have been deleted.
+/// This is currently stored in [`Bob`] which is stored on the [`Context`], thus Bob can
+/// only run one securejoin joiner protocol at a time.
+///
+/// This purposefully has nothing optional, the state is always fully valid.  See
+/// [`Bob::state`] to get access to this state.
 ///
 /// # Conducting the securejoin handshake
 ///
@@ -67,8 +177,6 @@ pub enum BobHandshakeStage {
 /// [`Bob::state`]: super::Bob::state
 #[derive(Debug, Clone)]
 pub struct BobState {
-    /// Database primary key.
-    id: i64,
     /// The QR Invite code.
     invite: QrInvite,
     /// The next expected message from Alice.
@@ -80,120 +188,39 @@ pub struct BobState {
 impl BobState {
     /// Starts the securejoin protocol and creates a new [`BobState`].
     ///
-    /// The `chat_id` needs to be the ID of the 1:1 chat with Alice, this chat will be used
-    /// to exchange the SecureJoin handshake messages as well as for showing error messages.
-    ///
     /// # Bob - the joiner's side
     /// ## Step 2 in the "Setup Contact protocol", section 2.1 of countermitm 0.10.0
-    ///
-    /// This currently aborts any other securejoin process if any did not yet complete.  The
-    /// ChatIds of the relevant 1:1 chat of any aborted handshakes are returned so that you
-    /// can report the aboreted handshake in the chat.  (Yes, there can only ever be one
-    /// ChatId in that Vec, the database doesn't care though.)
     pub async fn start_protocol(
         context: &Context,
         invite: QrInvite,
-        chat_id: ChatId,
-    ) -> Result<(Self, BobHandshakeStage, Vec<Self>), JoinError> {
-        let (stage, next) =
-            if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await?
-            {
-                // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
-                info!(context, "Taking securejoin protocol shortcut");
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::RequestWithAuth)
-                    .await?;
-                (
-                    BobHandshakeStage::RequestWithAuthSent,
-                    SecureJoinStep::ContactConfirm,
-                )
-            } else {
-                send_handshake_message(context, &invite, chat_id, BobHandshakeMsg::Request).await?;
-                (BobHandshakeStage::RequestSent, SecureJoinStep::AuthRequired)
+    ) -> Result<(Self, BobHandshakeStage), JoinError> {
+        let chat_id =
+            ChatId::create_for_contact_with_blocked(context, invite.contact_id(), Blocked::Yes)
+                .await
+                .map_err(JoinError::UnknownContact)?;
+        if fingerprint_equals_sender(context, invite.fingerprint(), invite.contact_id()).await? {
+            // The scanned fingerprint matches Alice's key, we can proceed to step 4b.
+            info!(context, "Taking securejoin protocol shortcut");
+            let state = Self {
+                invite,
+                next: SecureJoinStep::ContactConfirm,
+                chat_id,
             };
-        let (id, aborted_states) =
-            Self::insert_new_db_entry(context, next, invite.clone(), chat_id).await?;
-        let state = Self {
-            id,
-            invite,
-            next,
-            chat_id,
-        };
-        Ok((state, stage, aborted_states))
-    }
-
-    /// Inserts a new entry in the bobstate table, deleting all previous entries.
-    ///
-    /// Returns the ID of the newly inserted entry and all the aborted states.
-    async fn insert_new_db_entry(
-        context: &Context,
-        next: SecureJoinStep,
-        invite: QrInvite,
-        chat_id: ChatId,
-    ) -> Result<(i64, Vec<Self>)> {
-        context
-            .sql
-            .transaction(move |transaction| {
-                // We need to start a write transaction right away, so that we have the
-                // database locked and no one else can write to this table while we read the
-                // rows that we will delete.  So start with a dummy UPDATE.
-                transaction.execute(
-                    r#"UPDATE bobstate SET next_step=?;"#,
-                    params![SecureJoinStep::Terminated],
-                )?;
-                let mut stmt = transaction.prepare("SELECT id FROM bobstate;")?;
-                let mut aborted = Vec::new();
-                for id in stmt.query_map(params![], |row| row.get::<_, i64>(0))? {
-                    let id = id?;
-                    let state = BobState::from_db_id(transaction, id)?;
-                    aborted.push(state);
-                }
-
-                // Finally delete everything and insert new row.
-                transaction.execute("DELETE FROM bobstate;", params![])?;
-                transaction.execute(
-                    "INSERT INTO bobstate (invite, next_step, chat_id) VALUES (?, ?, ?);",
-                    params![invite, next, chat_id],
-                )?;
-                let id = transaction.last_insert_rowid();
-                Ok((id, aborted))
-            })
-            .await
-    }
-
-    /// Load [`BobState`] from the database.
-    pub async fn from_db(sql: &Sql) -> Result<Option<Self>> {
-        // Because of how Self::start_protocol() updates the database we are currently
-        // guaranteed to only have one row.
-        sql.query_row_optional(
-            "SELECT id, invite, next_step, chat_id FROM bobstate;",
-            paramsv![],
-            |row| {
-                let s = BobState {
-                    id: row.get(0)?,
-                    invite: row.get(1)?,
-                    next: row.get(2)?,
-                    chat_id: row.get(3)?,
-                };
-                Ok(s)
-            },
-        )
-        .await
-    }
-
-    fn from_db_id(connection: &Connection, id: i64) -> rusqlite::Result<Self> {
-        connection.query_row(
-            "SELECT invite, next_step, chat_id FROM bobstate WHERE id=?;",
-            params![id],
-            |row| {
-                let s = BobState {
-                    id,
-                    invite: row.get(0)?,
-                    next: row.get(1)?,
-                    chat_id: row.get(2)?,
-                };
-                Ok(s)
-            },
-        )
+            state
+                .send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestWithAuthSent))
+        } else {
+            let state = Self {
+                invite,
+                next: SecureJoinStep::AuthRequired,
+                chat_id,
+            };
+            state
+                .send_handshake_message(context, BobHandshakeMsg::Request)
+                .await?;
+            Ok((state, BobHandshakeStage::RequestSent))
+        }
     }
 
     /// Returns the [`QrInvite`] used to create this [`BobState`].
@@ -201,45 +228,20 @@ impl BobState {
         &self.invite
     }
 
-    /// Returns the [`ChatId`] of the 1:1 chat with the inviter (Alice).
-    pub fn alice_chat(&self) -> ChatId {
-        self.chat_id
-    }
-
-    /// Updates the [`BobState::next`] field in memory and the database.
-    ///
-    /// If the next state is a terminal state it will remove this [`BobState`] from the
-    /// database.
-    ///
-    /// If a user scanned a new QR code after this [`BobState`] was loaded this update will
-    /// fail currently because starting a new joiner process currently kills any previously
-    /// running processes.  This is a limitation which will go away in the future.
-    async fn update_next(&mut self, sql: &Sql, next: SecureJoinStep) -> Result<()> {
-        // TODO: write test verifying how this would fail.
-        match next {
-            SecureJoinStep::AuthRequired | SecureJoinStep::ContactConfirm => {
-                sql.execute(
-                    "UPDATE bobstate SET next_step=? WHERE id=?;",
-                    paramsv![next, self.id],
-                )
-                .await?;
-            }
-            SecureJoinStep::Terminated | SecureJoinStep::Completed => {
-                sql.execute("DELETE FROM bobstate WHERE id=?;", paramsv!(self.id))
-                    .await?;
-            }
-        }
-        self.next = next;
-        Ok(())
-    }
-
     /// Handles the given message for the securejoin handshake for Bob.
     ///
     /// If the message was not used for this handshake `None` is returned, otherwise the new
     /// stage is returned.  Once [`BobHandshakeStage::Completed`] or
     /// [`BobHandshakeStage::Terminated`] are reached this [`BobState`] should be destroyed,
     /// further calling it will just result in the messages being unused by this handshake.
-    pub async fn handle_message(
+    ///
+    /// # Errors
+    ///
+    /// Under normal operation this should never return an error, regardless of what kind of
+    /// message it is called with.  Any errors therefore should be treated as fatal internal
+    /// errors and this entire [`BobState`] should be thrown away as the state machine can
+    /// no longer be considered consistent.
+    async fn handle_message(
         &mut self,
         context: &Context,
         mime_message: &MimeMessage,
@@ -302,20 +304,17 @@ impl BobState {
             } else {
                 "Required encryption missing"
             };
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
+            self.next = SecureJoinStep::Terminated;
             return Ok(Some(BobHandshakeStage::Terminated(reason)));
         }
         if !fingerprint_equals_sender(context, self.invite.fingerprint(), self.invite.contact_id())
             .await?
         {
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
+            self.next = SecureJoinStep::Terminated;
             return Ok(Some(BobHandshakeStage::Terminated("Fingerprint mismatch")));
         }
         info!(context, "Fingerprint verified.",);
-        self.update_next(&context.sql, SecureJoinStep::ContactConfirm)
-            .await?;
+        self.next = SecureJoinStep::ContactConfirm;
         self.send_handshake_message(context, BobHandshakeMsg::RequestWithAuth)
             .await?;
         Ok(Some(BobHandshakeStage::RequestWithAuthSent))
@@ -363,8 +362,7 @@ impl BobState {
         if vg_expect_encrypted
             && !encrypted_and_signed(context, mime_message, Some(self.invite.fingerprint()))
         {
-            self.update_next(&context.sql, SecureJoinStep::Terminated)
-                .await?;
+            self.next = SecureJoinStep::Terminated;
             return Ok(Some(BobHandshakeStage::Terminated(
                 "Contact confirm message not encrypted",
             )));
@@ -396,8 +394,7 @@ impl BobState {
             // This is not an error affecting the protocol outcome.
             .ok();
 
-        self.update_next(&context.sql, SecureJoinStep::Completed)
-            .await?;
+        self.next = SecureJoinStep::Completed;
         Ok(Some(BobHandshakeStage::Completed))
     }
 
@@ -409,60 +406,48 @@ impl BobState {
         context: &Context,
         step: BobHandshakeMsg,
     ) -> Result<(), SendMsgError> {
-        send_handshake_message(context, &self.invite, self.chat_id, step).await
+        let mut msg = Message {
+            viewtype: Viewtype::Text,
+            text: Some(step.body_text(&self.invite)),
+            hidden: true,
+            ..Default::default()
+        };
+        msg.param.set_cmd(SystemMessage::SecurejoinMessage);
+
+        // Sends the step in Secure-Join header.
+        msg.param
+            .set(Param::Arg, step.securejoin_header(&self.invite));
+
+        match step {
+            BobHandshakeMsg::Request => {
+                // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.invitenumber());
+                msg.force_plaintext();
+            }
+            BobHandshakeMsg::RequestWithAuth => {
+                // Sends the Secure-Join-Auth header in mimefactory.rs.
+                msg.param.set(Param::Arg2, self.invite.authcode());
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+            BobHandshakeMsg::ContactConfirmReceived => {
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
+            }
+        };
+
+        // Sends our own fingerprint in the Secure-Join-Fingerprint header.
+        let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
+        msg.param.set(Param::Arg3, bob_fp.hex());
+
+        // Sends the grpid in the Secure-Join-Group header.
+        if let QrInvite::Group { ref grpid, .. } = self.invite {
+            msg.param.set(Param::Arg4, grpid);
+        }
+
+        chat::send_msg(context, self.chat_id, &mut msg).await?;
+        Ok(())
     }
 }
 
-/// Sends the requested handshake message to Alice.
-///
-/// Same as [`BobState::send_handshake_message`] but this variation allows us to send this
-/// message before we create the state in [`BobState::start_protocol`].
-async fn send_handshake_message(
-    context: &Context,
-    invite: &QrInvite,
-    chat_id: ChatId,
-    step: BobHandshakeMsg,
-) -> Result<(), SendMsgError> {
-    let mut msg = Message {
-        viewtype: Viewtype::Text,
-        text: Some(step.body_text(invite)),
-        hidden: true,
-        ..Default::default()
-    };
-    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
-
-    // Sends the step in Secure-Join header.
-    msg.param.set(Param::Arg, step.securejoin_header(invite));
-
-    match step {
-        BobHandshakeMsg::Request => {
-            // Sends the Secure-Join-Invitenumber header in mimefactory.rs.
-            msg.param.set(Param::Arg2, invite.invitenumber());
-            msg.force_plaintext();
-        }
-        BobHandshakeMsg::RequestWithAuth => {
-            // Sends the Secure-Join-Auth header in mimefactory.rs.
-            msg.param.set(Param::Arg2, invite.authcode());
-            msg.param.set_int(Param::GuaranteeE2ee, 1);
-        }
-        BobHandshakeMsg::ContactConfirmReceived => {
-            msg.param.set_int(Param::GuaranteeE2ee, 1);
-        }
-    };
-
-    // Sends our own fingerprint in the Secure-Join-Fingerprint header.
-    let bob_fp = SignedPublicKey::load_self(context).await?.fingerprint();
-    msg.param.set(Param::Arg3, bob_fp.hex());
-
-    // Sends the grpid in the Secure-Join-Group header.
-    if let QrInvite::Group { ref grpid, .. } = invite {
-        msg.param.set(Param::Arg4, grpid);
-    }
-
-    chat::send_msg(context, chat_id, &mut msg).await?;
-    Ok(())
-}
-
 /// Identifies the SecureJoin handshake messages Bob can send.
 enum BobHandshakeMsg {
     /// vc-request or vg-request
@@ -507,8 +492,8 @@ impl BobHandshakeMsg {
 }
 
 /// The next message expected by [`BobState`] in the setup-contact/secure-join protocol.
-#[derive(Debug, Clone, Copy, PartialEq)]
-pub enum SecureJoinStep {
+#[derive(Debug, Clone, PartialEq)]
+enum SecureJoinStep {
     /// Expecting the auth-required message.
     ///
     /// This corresponds to the `vc-auth-required` or `vg-auth-required` message of step 3d.
@@ -548,29 +533,3 @@ impl SecureJoinStep {
         }
     }
 }
-
-impl rusqlite::types::ToSql for SecureJoinStep {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
-        let num = match &self {
-            SecureJoinStep::AuthRequired => 0,
-            SecureJoinStep::ContactConfirm => 1,
-            SecureJoinStep::Terminated => 2,
-            SecureJoinStep::Completed => 3,
-        };
-        let val = rusqlite::types::Value::Integer(num);
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
-impl rusqlite::types::FromSql for SecureJoinStep {
-    fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| match val {
-            0 => Ok(SecureJoinStep::AuthRequired),
-            1 => Ok(SecureJoinStep::ContactConfirm),
-            2 => Ok(SecureJoinStep::Terminated),
-            3 => Ok(SecureJoinStep::Completed),
-            _ => Err(rusqlite::types::FromSqlError::OutOfRange(val)),
-        })
-    }
-}

src/securejoin/qrinvite.rs

@@ -8,23 +8,22 @@ use std::convert::TryFrom;
 
 use anyhow::{bail, Error, Result};
 
-use crate::contact::ContactId;
 use crate::key::Fingerprint;
 use crate::qr::Qr;
 
 /// Represents the data from a QR-code scan.
 ///
 /// There are methods to conveniently access fields present in both variants.
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+#[derive(Debug, Clone)]
 pub enum QrInvite {
     Contact {
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         invitenumber: String,
         authcode: String,
     },
     Group {
-        contact_id: ContactId,
+        contact_id: u32,
         fingerprint: Fingerprint,
         name: String,
         grpid: String,
@@ -38,7 +37,7 @@ impl QrInvite {
     ///
     /// The actual QR-code contains a URL-encoded email address, but upon scanning this is
     /// translated to a contact ID.
-    pub fn contact_id(&self) -> ContactId {
+    pub fn contact_id(&self) -> u32 {
         match self {
             Self::Contact { contact_id, .. } | Self::Group { contact_id, .. } => *contact_id,
         }
@@ -101,22 +100,3 @@ impl TryFrom<Qr> for QrInvite {
         }
     }
 }
-
-impl rusqlite::types::ToSql for QrInvite {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
-        let json = serde_json::to_string(self)
-            .map_err(|err| rusqlite::Error::ToSqlConversionFailure(Box::new(err)))?;
-        let val = rusqlite::types::Value::Text(json);
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
-impl rusqlite::types::FromSql for QrInvite {
-    fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
-        String::column_result(value).and_then(|val| {
-            serde_json::from_str(&val)
-                .map_err(|err| rusqlite::types::FromSqlError::Other(Box::new(err)))
-        })
-    }
-}

src/simplify.rs

@@ -240,7 +240,7 @@ fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
         ret += " [...]";
     }
     // redo escaping done by escape_message_footer_marks()
-    ret.replace('\u{200B}', "")
+    ret.replace("\u{200B}", "")
 }
 
 /// Returns true if the line contains only whitespace.