Release 1.60.0

Debian deprecated `which` in `debianutils` in favor of `command`.

`which` outputs this to stderr now:
/usr/bin/which: this version of `which' is deprecated; use `command -v' in scripts instead.

.circleci/config.yml

@@ -1,68 +0,0 @@
-version: 2.1
-executors:
-  default:
-    docker:
-      - image: filecoin/rust:latest
-    working_directory: /mnt/crate
-  doxygen:
-    docker:
-      - image: hrektts/doxygen
-            
-jobs:
-  build_doxygen:
-    executor: doxygen
-    steps:
-      - checkout
-      - run: bash scripts/run-doxygen.sh
-      - run: mkdir -p workspace/c-docs
-      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - c-docs
-
-  remote_python_packaging:
-    machine: true
-    steps:
-      - checkout
-      # the following commands on success produces
-      # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash scripts/remote_python_packaging.sh
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
-
-  upload_docs_wheels:
-    machine: true
-    steps:
-      - checkout
-      - attach_workspace:
-          at: workspace
-      - run: ls -laR workspace
-      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
-
-workflows:
-  version: 2.1
-
-  test:
-    jobs:
-      - remote_python_packaging:
-          filters:
-            branches:
-              only: master
-
-      - upload_docs_wheels:
-          requires:
-             - remote_python_packaging
-             - build_doxygen
-          filters:
-            branches:
-              only: master
-
-      - build_doxygen:
-          filters:
-            branches:
-              only: master

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "cargo"
+    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.50.0
+          toolchain: stable
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.50.0
+            toolchain: stable
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -65,26 +65,27 @@ jobs:
 
   build_and_test:
     name: Build and test
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        # macOS disabled due to random failures related to caching
-        #os: [ubuntu-latest, windows-latest, macOS-latest]
-        os: [ubuntu-latest, windows-latest]
-        rust: [1.50.0]
-        experimental: [false]
-#        include:
-#          - os: ubuntu-latest
-#            rust: nightly
-#            experimental: true
-#          - os: windows-latest
-#            rust: nightly
-#            experimental: true
-#          - os: macOS-latest
-#            rust: nightly
-#            experimental: true
+        include:
+          # Currently used Rust version, same as in `rust-toolchain` file.
+          - os: ubuntu-latest
+            rust: 1.54.0
+            python: 3.9
+          - os: windows-latest
+            rust: 1.54.0
+            python: false # Python bindings compilation on Windows is not supported.
 
+          # Minimum Supported Rust Version = 1.48.0
+          # This is the Debian "bullseye" release version of Rust.
+          #
+          # Minimum Supported Python Version = 3.7
+          # This is the minimum version for which manylinux Python wheels are
+          # built.
+          - os: ubuntu-latest
+            rust: 1.48.0
+            python: 3.7
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -114,8 +115,10 @@ jobs:
 
     - name: check
       uses: actions-rs/cargo@v1
+      env:
+        RUSTFLAGS: -D warnings
       with:
-        command:  check
+        command: check
         args: --all --bins --examples --tests --features repl
 
     - name: tests
@@ -123,3 +126,29 @@ jobs:
       with:
         command: test
         args: --all
+
+    - name: install python
+      if: ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: install tox
+      if: ${{ matrix.python }}
+      run: pip install tox
+
+    - name: build C library
+      if: ${{ matrix.python }}
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: -p deltachat_ffi
+
+    - name: run python tests
+      if: ${{ matrix.python }}
+      env:
+        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        DCC_RS_TARGET: debug
+        DCC_RS_DEV: ${{ github.workspace }}
+      working-directory: python
+      run: tox -e lint,doc,py3

.github/workflows/remote_tests.yml

@@ -1,21 +0,0 @@
-name: Remote tests
-on: [push]
-jobs:
-  remote_tests_python:
-    name: Remote Python tests
-    runs-on: ubuntu-latest
-    env:
-      CIRCLE_BRANCH: ${{ github.ref }}
-      CIRCLE_JOB: remote_tests_python
-      CIRCLE_BUILD_NUM: ${{ github.run_number }}
-      DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-    steps:
-      - uses: actions/checkout@v2
-      - run: mkdir -m 700 -p ~/.ssh
-      - run: touch ~/.ssh/id_ed25519
-      - run: chmod 600 ~/.ssh/id_ed25519
-      - run: 'echo "$SSH_KEY" | base64 -d > ~/.ssh/id_ed25519'
-        shell: bash
-        env:
-          SSH_KEY: ${{ secrets.SSH_KEY }}
-      - run: scripts/remote_tests_python.sh

.github/workflows/repl.yml

@@ -0,0 +1,32 @@
+# Manually triggered action to build a Windows repl.exe which users can
+# download to debug complex bugs.
+
+name: Build Windows REPL .exe
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_repl:
+    name: Build REPL example
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.50.0
+        override: true
+
+    - name: build
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --example repl --features repl,vendored
+
+    - name: Upload binary
+      uses: actions/upload-artifact@v2
+      with:
+        name: repl.exe
+        path: 'target/debug/examples/repl.exe'

CHANGELOG.md

@@ -1,5 +1,213 @@
 # Changelog 
 
+## 1.60.0
+
+### Added
+- add device message to warn about QUOTA #2621
+- add SOCKS5 support #2474 #2620
+
+### Changes
+- don't emit multiple events with the same import/export progress number #2639
+- reduce message length limit to 5000 chars #2615
+
+### Fixes
+- keep event emitter from closing when there are no accounts #2636
+
+## 1.59.0
+
+### Added
+- add quota information to `dc_get_connectivity_html()`
+
+### Changes
+- refactorings #2592 #2570 #2581
+- add 'device chat about' to now existing status #2613
+- update provider database #2608
+
+### Fixes
+- provider database supports socket=PLAIN and dotless domains now #2604 #2608
+- add migrated accounts to events emitter #2607
+- fix forwarding quote-only mails #2600
+- do not set WantsMdn param for outgoing messages #2603
+- set timestamps for system messages #2593
+- do not treat gmail labels as folders #2587
+- avoid timing problems in `dc_maybe_network_lost()` #2551
+- only set smtp to "connected" if the last message was actually sent #2541
+
+
+## 1.58.0
+
+### Fixes
+
+- move WAL file together with database
+  and avoid using data if the database was not closed correctly before #2583
+
+
+## 1.57.0
+
+### API Changes
+
+- breaking change: removed deaddrop chat #2514 #2563
+
+  Contact request chats are not merged into a single virtual
+  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
+  same way as other chats, but sending of messages to them is not
+  allowed and MDNs are not sent automatically until the chat is
+  "accepted" by the user.
+
+  New API:
+  - `dc_chat_is_contact_request()`: returns true if chat is a contact
+    request.  In this case an option to accept the chat via
+    `dc_accept_chat()` should be shown in the UI.
+  - `dc_accept_chat()`: unblock the chat or accept contact request
+  - `dc_block_chat()`: block the chat, currently works only for mailing
+    lists.
+
+  Removed API:
+  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
+    `dc_decide_on_contact_request()`
+  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
+    `dc_decide_on_contact_request()`
+  - `dc_decide_on_contact_request()`: this call requires a message ID
+    from deaddrop chat as input. As deaddrop chat is removed, this
+    call can't be used anymore.
+  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
+    only difference between these calls was in handling of deaddrop
+    chat
+  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
+
+- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
+  Instead, there is a new api `dc_get_connectivity()`
+  and `dc_get_connectivity_html()`;
+  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
+
+- breaking change: removed `dc_accounts_import_account()`
+  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
+  on its context
+
+- update account api, 2 new methods:
+  `int dc_all_work_done (dc_context_t* context);`
+  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
+
+- add api to check if a message was `Auto-Submitted`
+  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
+  python: `Message.is_bot()`
+
+- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
+  now returns `NULL` if there is no selected account
+
+- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
+  connectivity loss on its own (eg. iOS) #2550
+
+### Added
+- use Auto-Submitted: auto-generated header to identify bots #2502
+- allow sending stickers via repl tool
+- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
+- withdraw/revive own qr-codes #2512
+- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
+
+### Changes
+- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
+- update documentation #2548 #2561 #2569
+- breaking: `Accounts::create` does not also create an default account anymore #2500
+- remove "forwarded" from stickers, as the primary way of getting stickers
+  is by asking a bot and then forwarding them currently #2526
+- mimeparser: use mailparse to parse RFC 2231 filenames #2543
+- allow email addresses without dot in the domain part #2112
+- allow installing lib and include under different prefixes #2558
+- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
+- improve tests #2487 #2491 #2497
+- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
+- improve ci #2494
+- update provider-database #2565
+
+### Removed
+- remove `dc_accounts_import_account()` api #2521
+- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
+
+### Fixes
+- allow stickers with gif-images #2481
+- fix database migration #2486
+- do not count hidden messages in get_msg_cnt(). #2493
+- improve drafts detection #2489
+- fix panic when removing last, selected account from account manager #2500
+- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
+- avoid hiding outgoing classic emails #2505
+- fixes for message timestamps #2517
+- do not process names, avatars, location XMLs, message signature etc.
+  for duplicate messages #2513
+- fix `can_send` for users not in group #2479
+- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
+- fix which chats messages are assigned to #2465
+- fix: don't create chats when MDNs are received #2578
+
+
+## 1.56.0
+
+- fix downscaling images #2469
+
+- fix outgoing messages popping up in selfchat #2456
+
+- securejoin: display error reason if there is any #2470
+
+- do not allow deleting contacts with ongoing chats #2458
+
+- fix: ignore drafts folder when scanning #2454
+
+- fix: scan folders also when inbox is not watched #2446
+
+- more robust In-Reply-To parsing #2182
+
+- update dependencies #2441 #2438 #2439 #2440 #2447 #2448 #2449 #2452 #2453 #2460 #2464 #2466
+
+- update provider-database #2471
+
+- refactorings #2459 #2457
+
+- improve tests and ci #2445 #2450 #2451
+
+
+## 1.55.0
+
+- fix panic when receiving some HTML messages #2434
+
+- fix downloading some messages multiple times #2430
+
+- fix formatting of read receipt texts #2431
+
+- simplify SQL error handling #2415
+
+- explicit rust API for creating chats with blocked status #2282
+
+- debloat the binary by using less AsRef arguments #2425
+
+
+## 1.54.0
+
+- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
+
+- global search performance improvement #2364 #2365 #2366
+
+- improve SQLite performance with `PRAGMA synchronous=normal` #2382
+
+- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
+
+- python: list `requests` as a requirement #2390
+
+- fix creation of many delete jobs when being offline #2372
+
+- synchronize status between devices #2386
+
+- deaddrop (contact requests) chat improvements #2373
+
+- add "Forwarded:" to notification and chatlist summaries #2310
+
+- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
+
+- improve tests #2360 #2362 #2370 #2377 #2387
+
+- cleanup #2359 #2361 #2374 #2376 #2379 #2388
+
+
 ## 1.53.0
 
 - fix sqlx performance regression #2355 2356

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(deltachat)
+project(deltachat LANGUAGES C)
 
 find_program(CARGO cargo)
 
@@ -8,8 +8,22 @@ add_custom_command(
 	"target/release/libdeltachat.a"
 	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
-	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --package deltachat_ffi --release
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        COMMAND
+        PREFIX=${CMAKE_INSTALL_PREFIX}
+        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
+        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
+        ${CARGO} build --release --no-default-features
+
+	# Build in `deltachat-ffi` directory instead of using
+	# `--package deltachat_ffi` to avoid feature resolver version
+	# "1" bug which makes `--no-default-features` affect only
+	# `deltachat`, but not `deltachat-ffi` package.
+	#
+	# We can't enable version "2" resolver [1] because it is not
+	# stable yet on rust 1.50.0.
+	#
+	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 
 add_custom_target(

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.53.0"
+version = "1.60.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,80 +12,84 @@ debug = 0
 lto = true
 
 [dependencies]
+deltachat_derive = { path = "./deltachat_derive" }
+
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.28"
-async-imap = "0.4.0"
+anyhow = "1.0.42"
+async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = { version = "0.3.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
-async-std-resolver = "0.19.5"
-async-std = { version = "~1.8.0", features = ["unstable"] }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
+async-std-resolver = "0.20.3"
+async-std = { version = "~1.9.0", features = ["unstable"] }
 async-tar = "0.3.0"
-async-trait = "0.1.31"
-backtrace = "0.3.33"
+async-trait = "0.1.50"
+backtrace = "0.3.59"
 base64 = "0.13"
-bitflags = "1.1.0"
+bitflags = "1.3.1"
 byteorder = "1.3.1"
-charset = "0.1"
 chrono = "0.4.6"
-dirs = { version = "3.0.1", optional=true }
+dirs = { version = "3.0.2", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.0"
-futures = "0.3.4"
+escaper = "0.1.1"
+futures = "0.3.16"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.3.0"
-itertools = "0.10.0"
+indexmap = "1.7.0"
+itertools = "0.10.1"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.51"
+libc = "0.2.98"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13.0"
+mailparse = "0.13.5"
 native-tls = "0.2.3"
 num_cpus = "1.13.0"
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-once_cell = "1.4.1"
+once_cell = "1.8.0"
 percent-encoding = "2.0"
 pgp = { version = "0.7.0", default-features = false }
 pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.18.1"
+quick-xml = "0.22.0"
+r2d2 = "0.8.9"
+r2d2_sqlite = "0.18.0"
 rand = "0.7.0"
-regex = "1.1.6"
+regex = "1.4.6"
+rusqlite = "0.25"
 rust-hsluv = "0.1.4"
-rustyline = { version = "4.1.0", optional = true }
+rustyline = { version = "8.2.0", optional = true }
 sanitize-filename = "0.3.0"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.3"
-sha2 = "0.9.0"
+sha-1 = "0.9.7"
+sha2 = "0.9.5"
 smallvec = "1.0.0"
-sqlx = { git = "https://github.com/deltachat/sqlx", branch = "master", features = ["runtime-async-std-native-tls", "sqlite"] }
-# keep in sync with sqlx
-libsqlite3-sys = { version = "0.22.0", default-features = false, features = [ "pkg-config", "vcpkg", "bundled" ] }
-stop-token = { version = "0.1.1", features = ["unstable"] }
-strum = "0.20.0"
-strum_macros = "0.20.1"
+stop-token = "0.2.0"
+strum = "0.21.0"
+strum_macros = "0.21.1"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
-thiserror = "1.0.14"
+thiserror = "1.0.26"
 toml = "0.5.6"
-url = "2.1.1"
+url = "2.2.2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+fast-socks5 = "0.4.2"
+humansize = "1.1.1"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
 criterion = "0.3"
-futures-lite = "1.7.0"
+futures-lite = "1.12.0"
 log = "0.4.11"
-pretty_assertions = "0.6.1"
+pretty_assertions = "0.7.2"
 pretty_env_logger = "0.4.0"
-proptest = "0.10"
+proptest = "1.0"
 tempfile = "3.0"
 
 [workspace]
 members = [
   "deltachat-ffi",
+  "deltachat_derive",
 ]
 
 [[example]]
@@ -112,8 +116,8 @@ name = "search_msgs"
 harness = false
 
 [features]
-default = []
+default = ["vendored"]
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
 nightly = ["pgp/nightly"]

README.md

@@ -3,8 +3,6 @@
 > Deltachat-core written in Rust 
 
 [![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
-[![Remote tests](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml)
-[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -81,6 +79,16 @@ For more commands type:
 > help
 ```
 
+## Installing libdeltachat system wide
+
+```
+$ git clone https://github.com/deltachat/deltachat-core-rust.git
+$ cd deltachat-core-rust
+$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
+$ cmake --build build
+$ sudo cmake --install build
+```
+
 ## Development
 
 ```sh

benches/contacts.rs

@@ -17,7 +17,7 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
         .collect::<Vec<String>>()
         .join("");
 
-    Contact::add_address_book(&context, book).await.unwrap();
+    Contact::add_address_book(&context, &book).await.unwrap();
 
     let query: Option<&str> = None;
     for _ in 0..read_count {

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.53.0"
+version = "1.60.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,9 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.6.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
+async-std = "1.9.0"
+anyhow = "1.0.42"
+thiserror = "1.0.26"
 rand = "0.7.3"
 
 [features]

deltachat-ffi/build.rs

@@ -23,11 +23,13 @@ fn main() {
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
         prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
+        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
+        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(&pkg_config.as_bytes())
+        .write_all(pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.h

@@ -271,6 +271,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_port`    = SMTP-port, guessed if left out
  * - `send_security`= SMTP-socket, one of @ref DC_SOCKET, defaults to #DC_SOCKET_AUTO
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
+ * - `socks5_enabled` = SOCKS5 enabled
+ * - `socks5_host` = SOCKS5 proxy server host
+ * - `socks5_port` = SOCKS5 proxy server port
+ * - `socks5_user` = SOCKS5 proxy username
+ * - `socks5_password` = SOCKS5 proxy password
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way e.g. using CC, defaults to empty
@@ -305,7 +310,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=
  *                    also show all mails of confirmed contacts,
  *                    DC_SHOW_EMAILS_ALL (2)=
- *                    also show mails of unconfirmed contacts in the deaddrop.
+ *                    also show mails of unconfirmed contacts.
  * - `key_gen_type` = DC_KEY_GEN_DEFAULT (0)=
  *                    generate recommended key type (default),
  *                    DC_KEY_GEN_RSA2048 (1)=
@@ -340,7 +345,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    https://github.com/cracker0dks/basicwebrtc which some UIs have native support for.
  *                    The type `jitsi:` may be handled by external apps.
  *                    If no type is prefixed, the videochat is handled completely in a browser.
- * - `bot`          = Set to "1" if this is a bot. E.g. prevents adding the "Device messages" and "Saved messages" chats.
+ * - `bot`          = Set to "1" if this is a bot.
+ *                    Prevents adding the "Device messages" and "Saved messages" chats,
+ *                    adds Auto-Submitted header to outgoing messages.
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
@@ -370,6 +377,9 @@ int             dc_set_config                (dc_context_t* context, const char*
  *                    an error (no warning as it should be shown to the user) is logged but the attachment is sent anyway.
  * - `sys.config_keys` = get a space-separated list of all config-keys available.
  *                    The config-keys are the keys that can be passed to the parameter `key` of this function.
+ * - `quota_exceeding` = 0: quota is unknown or in normal range;
+ *                    >=80: quota is about to exceed, the value is the concrete percentage,
+ *                    a device message is added when that happens, however, that value may still be interesting for bots.
  *
  * @memberof dc_context_t
  * @param context The context object. For querying system values, this can be NULL.
@@ -461,11 +471,66 @@ char*           dc_get_info                  (const dc_context_t* context);
 char*           dc_get_oauth2_url            (dc_context_t* context, const char* addr, const char* redirect_uri);
 
 
+#define DC_CONNECTIVITY_NOT_CONNECTED        1000
+#define DC_CONNECTIVITY_CONNECTING           2000
+#define DC_CONNECTIVITY_WORKING              3000
+#define DC_CONNECTIVITY_CONNECTED            4000
+
+
+/**
+ * Get the current connectivity, i.e. whether the device is connected to the IMAP server.
+ * One of:
+ * - DC_CONNECTIVITY_NOT_CONNECTED (1000-1999): Show e.g. the string "Not connected" or a red dot
+ * - DC_CONNECTIVITY_CONNECTING (2000-2999): Show e.g. the string "Connecting…" or a yellow dot
+ * - DC_CONNECTIVITY_WORKING (3000-3999): Show e.g. the string "Getting new messages" or a spinning wheel
+ * - DC_CONNECTIVITY_CONNECTED (>=4000): Show e.g. the string "Connected" or a green dot
+ *
+ * We don't use exact values but ranges here so that we can split up
+ * states into multiple states in the future.
+ *
+ * Meant as a rough overview that can be shown 
+ * e.g. in the title of the main screen.
+ *
+ * If the connectivity changes, a #DC_EVENT_CONNECTIVITY_CHANGED will be emitted.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return The current connectivity.
+ */
+int             dc_get_connectivity          (dc_context_t* context);
+
+
+/**
+ * Get an overview of the current connectivity, and possibly more statistics.
+ * Meant to give the user more insight about the current status than
+ * the basic connectivity info returned by dc_get_connectivity(); show this
+ * e.g., if the user taps on said basic connectivity info.
+ *
+ * If this page changes, a #DC_EVENT_CONNECTIVITY_CHANGED will be emitted.
+ *
+ * This comes as an HTML from the core so that we can easily improve it
+ * and the improvement instantly reaches all UIs.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return An HTML page with some info about the current connectivity and status.
+ */
+char*           dc_get_connectivity_html     (dc_context_t* context);
+
+
+/**
+ * Standalone version of dc_accounts_all_work_done().
+ * Only used by the python tests.
+ */
+int             dc_all_work_done             (dc_context_t* context);
+
+
 // connect
 
 /**
  * Configure a context.
- * During configuration IO must not be started, if needed stop IO using dc_stop_io() first.
+ * During configuration IO must not be started,
+ * if needed stop IO using dc_accounts_stop_io() or dc_stop_io() first.
  * If the context is already configured,
  * this function will try to change the configuration.
  *
@@ -625,12 +690,6 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *
  * By default, the function adds some special entries to the list.
  * These special entries can be identified by the ID returned by dc_chatlist_get_chat_id():
- * - DC_CHAT_ID_DEADDROP (1) - this special chat is present if there are
- *   messages from addresses that have no relationship to the configured account.
- *   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
- *   about it with dc_chatlist_get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
- *   and offers the options "Start chat", "Block" or "Not now".
- *   Call dc_decide_on_contact_request() when the user selected one of these options.
  * - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
  *   archived _any_ chat using dc_set_chat_visibility(). The UI should show a link as
  *   "Show archived chats", if the user clicks this item, the UI should show a
@@ -648,10 +707,10 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *       the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
  *       chats
  *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
- *       and hides the "Device chat" and the deaddrop.
+ *       and hides the "Device chat" and contact requests.
  *       typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
  *       to also hide the archive link.
- *     - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
+ *     - if the flag DC_GCL_NO_SPECIALS is set, archive link is not added
  *       to the list (may be used e.g. for selecting chats on forwarding, the flag is
  *       not needed when DC_GCL_ARCHIVED_ONLY is already set)
  *     - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
@@ -671,42 +730,12 @@ dc_chatlist_t*  dc_get_chatlist              (dc_context_t* context, int flags,
 
 // handle chats
 
-/**
- * Create a normal chat or a group chat by a messages ID that comes typically
- * from the deaddrop, DC_CHAT_ID_DEADDROP (1).
- *
- * If the given message ID already belongs to a normal chat or to a group chat,
- * the chat ID of this chat is returned and no new chat is created.
- * If a new chat is created, the given message ID is moved to this chat, however,
- * there may be more messages moved to the chat from the deaddrop. To get the
- * chat messages, use dc_get_chat_msgs().
- *
- * If the user is asked before creation, he should be
- * asked whether he wants to chat with the _contact_ belonging to the message;
- * the group names may be really weird when taken from the subject of implicit
- * groups and this may look confusing.
- *
- * Moreover, this function also scales up the origin of the contact belonging
- * to the message and, depending on the contacts origin, messages from the
- * same group may be shown or not - so, all in all, it is fine to show the
- * contact name only.
- *
- * @deprecated Deprecated 2021-02-07, use dc_decide_on_contact_request() instead
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param msg_id The message ID to create the chat for.
- * @return The created or reused chat ID on success. 0 on errors.
- */
-uint32_t        dc_create_chat_by_msg_id     (dc_context_t* context, uint32_t msg_id);
-
-
 /**
  * Create a normal chat with a single user.  To create group chats,
  * see dc_create_group_chat().
  *
  * If a chat already exists, this ID is returned, otherwise a new chat is created;
- * this new chat may already contain messages, e.g. from the deaddrop, to get the
- * chat messages, use dc_get_chat_msgs().
+ * to get the chat messages, use dc_get_chat_msgs().
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
@@ -1078,9 +1107,9 @@ int             dc_estimate_deletion_cnt    (dc_context_t* context, int from_ser
  * or badge counters eg. on the app-icon.
  * The list is already sorted and starts with the most recent fresh message.
  *
- * Messages belonging to muted chats are not returned,
- * as they should not be notified
- * and also a badge counters should not include messages of muted chats.
+ * Messages belonging to muted chats or to the contact requests are not returned;
+ * these messages should not be notified
+ * and also badge counters should not include these messages.
  *
  * To get the number of fresh messages for a single chat, muted or not,
  * use dc_get_fresh_msg_cnt().
@@ -1213,6 +1242,31 @@ void            dc_set_chat_visibility       (dc_context_t* context, uint32_t ch
  */
 void            dc_delete_chat               (dc_context_t* context, uint32_t chat_id);
 
+/**
+ * Block a chat.
+ *
+ * Blocking 1:1 chats blocks the corresponding contact. Blocking
+ * mailing lists creates a pseudo-contact in the list of blocked
+ * contacts, so blocked mailing lists can be discovered and unblocked
+ * the same way as the contacts.  Blocking group chats deletes the
+ * chat without blocking any contacts, so it may pop up again later.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param chat_id The ID of the chat to block.
+ */
+void            dc_block_chat                (dc_context_t* context, uint32_t chat_id);
+
+/**
+ * Accept a contact request chat.
+ *
+ * Use it to accept "contact request" chats as indicated by dc_chat_is_contact_request().
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @param chat_id The ID of the chat to accept.
+ */
+void            dc_accept_chat               (dc_context_t* context, uint32_t chat_id);
 
 /**
  * Get contact IDs belonging to a chat.
@@ -1224,8 +1278,6 @@ void            dc_delete_chat               (dc_context_t* context, uint32_t ch
  *   explicitly as it may happen that oneself gets removed from a still existing
  *   group
  *
- * - for the deaddrop, the list is empty
- *
  * - for mailing lists, the behavior is not documented currently, we will decide on that later.
  *   for now, the UI should not show the list for mailing lists.
  *   (we do not know all members and there is not always a global mailing list address,
@@ -1574,32 +1626,22 @@ void            dc_forward_msgs              (dc_context_t* context, const uint3
 
 
 /**
- * Mark all messages sent by the given contact as _noticed_.
- * This function is typically used to ignore a user in the deaddrop temporarily ("Not now" button).
+ * Mark messages as presented to the user.
+ * Typically, UIs call this function on scrolling through the chatlist,
+ * when the messages are presented at least for a little moment.
+ * The concrete action depends on the type of the chat and on the users settings
+ * (dc_msgs_presented() may be a better name therefore, but well :)
  *
- * The contact is expected to belong to the deaddrop;
- * only one #DC_EVENT_MSGS_NOTICED with chat_id=DC_CHAT_ID_DEADDROP may be emitted.
+ * - For normal chats, the IMAP state is updated, MDN is sent
+ *   (if dc_set_config()-options `mdns_enabled` is set)
+ *   and the internal state is changed to DC_STATE_IN_SEEN to reflect these actions.
  *
- * See also dc_marknoticed_chat() and dc_markseen_msgs()
+ * - For contact requests, no IMAP or MDNs is done
+ *   and the internal state is not changed therefore.
+ *   See also dc_marknoticed_chat().
  *
- * @deprecated Deprecated 2021-02-07, use dc_decide_on_contact_request() if the user just hit "Not now" on a button in the deaddrop,
- *      dc_marknoticed_chat() if the user has entered a chat
- *      and dc_markseen_msgs() if the user actually _saw_ a message.
- * @memberof dc_context_t
- * @param context The context object.
- * @param contact_id The contact ID of which all messages should be marked as noticed.
- */
-void            dc_marknoticed_contact       (dc_context_t* context, uint32_t contact_id);
-
-
-/**
- * Mark a message as _seen_, updates the IMAP state and
- * sends MDNs. If the message is not in a real chat (e.g. a contact request), the
- * message is only marked as NOTICED and no IMAP/MDNs is done.  See also
- * dc_marknoticed_chat().
- *
- * Moreover, if messages belong to a chat with ephemeral messages enabled,
- * the ephemeral timer is started for these messages.
+ * Moreover, timer is started for incoming ephemeral messages.
+ * This also happens for contact requests chats.
  *
  * One #DC_EVENT_MSGS_NOTICED event is emitted per modified chat.
  *
@@ -1626,53 +1668,6 @@ void            dc_markseen_msgs             (dc_context_t* context, const uint3
 dc_msg_t*       dc_get_msg                   (dc_context_t* context, uint32_t msg_id);
 
 
-#define DC_DECISION_START_CHAT 0
-#define DC_DECISION_BLOCK      1
-#define DC_DECISION_NOT_NOW    2
-
-
-/**
- * Call this when the user decided about a deaddrop message ("Do you want to chat with NAME?").
- *
- * Possible decisions are:
- * - DC_DECISION_START_CHAT (0)
- *   - This will create a new chat and return the chat id.
- * - DC_DECISION_BLOCK (1)
- *   - This will block the sender.
- *   - When a new message from the sender arrives,
- *     that will not result in a new contact request.
- *   - The blocked sender will be returned by dc_get_blocked_contacts()
- *     typically, the UI offers an option to unblock senders from there.
- * - DC_DECISION_NOT_NOW (2)
- *   - This will mark all messages from this sender as noticed.
- *   - That the contact request is removed from the chat list.
- *   - When a new message from the sender arrives,
- *     a new contact request with the new message will pop up in the chatlist.
- *   - The contact request stays available in the explicit deaddrop.
- *   - If the contact request is already noticed, nothing happens.
- *
- * If the message belongs to a mailing list,
- * the function makes sure that all messages
- * from the mailing list are blocked or marked as noticed.
- *
- * The user should be asked whether they want to chat with the _contact_ belonging to the message;
- * the group names may be really weird when taken from the subject of implicit (= ad-hoc)
- * groups and this may look confusing. Moreover, this function also scales up the origin of the contact.
- *
- * If the chat belongs to a mailing list, you can also ask
- * "Would you like to read MAILING LIST NAME?"
- * (use dc_msg_get_real_chat_id() to get the chat-id for the contact request
- * and then dc_chat_is_mailing_list(), dc_chat_get_name() and so on)
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_id ID of Message to decide on.
- * @param decision One of the DC_DECISION_* values.
- * @return The chat id of the created chat, if any.
- */
-uint32_t        dc_decide_on_contact_request (dc_context_t* context, uint32_t msg_id, int decision);
-
-
 // handle contacts
 
 /**
@@ -1866,7 +1861,8 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
 
 /**
  * Import/export things.
- * During backup import/export IO must not be started, if needed stop IO using dc_stop_io() first.
+ * During backup import/export IO must not be started,
+ * if needed stop IO using dc_accounts_stop_io() or dc_stop_io() first.
  * What to do is defined by the _what_ parameter which may be one of the following:
  *
  * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`.
@@ -2064,27 +2060,80 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_TEXT                   330 // text1=text
 #define         DC_QR_URL                    332 // text1=URL
 #define         DC_QR_ERROR                  400 // text1=error string
+#define         DC_QR_WITHDRAW_VERIFYCONTACT 500
+#define         DC_QR_WITHDRAW_VERIFYGROUP   502 // text1=groupname
+#define         DC_QR_REVIVE_VERIFYCONTACT   510
+#define         DC_QR_REVIVE_VERIFYGROUP     512 // text1=groupname
 
 /**
  * Check a scanned QR code.
- * The function should be called after a QR code is scanned.
  * The function takes the raw text scanned and checks what can be done with it.
  *
+ * The UI is supposed to show the result to the user.
+ * In case there are further actions possible,
+ * the UI has to ask the user before doing further steps.
+ *
  * The QR code state is returned in dc_lot_t::state as:
  *
- * - DC_QR_ASK_VERIFYCONTACT with dc_lot_t::id=Contact ID
- * - DC_QR_ASK_VERIFYGROUP withdc_lot_t::text1=Group name
- * - DC_QR_FPR_OK with dc_lot_t::id=Contact ID
- * - DC_QR_FPR_MISMATCH with dc_lot_t::id=Contact ID
+ * - DC_QR_ASK_VERIFYCONTACT with dc_lot_t::id=Contact ID:
+ *   ask whether to verify the contact;
+ *   if so, start the protocol with dc_join_securejoin().
+ *
+ * - DC_QR_ASK_VERIFYGROUP withdc_lot_t::text1=Group name:
+ *   ask whether to join the group;
+ *   if so, start the protocol with dc_join_securejoin().
+ *
+ * - DC_QR_FPR_OK with dc_lot_t::id=Contact ID:
+ *   contact fingerprint verified,
+ *   ask the user if they want to start chatting;
+ *   if so, call dc_create_chat_by_contact_id().
+ *
+ * - DC_QR_FPR_MISMATCH with dc_lot_t::id=Contact ID:
+ *   scanned fingerprint does not match last seen fingerprint.
+ *
  * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::test1=Formatted fingerprint
- * - DC_QR_ACCOUNT allows creation of an account, dc_lot_t::text1=domain
- * - DC_QR_WEBRTC_INSTANCE - a shared webrtc-instance
- *   that will be set if dc_set_config_from_qr() is called with the qr-code,
- *   dc_lot_t::text1=domain could be used to ask the user
- * - DC_QR_ADDR with dc_lot_t::id=Contact ID
- * - DC_QR_TEXT with dc_lot_t::text1=Text
- * - DC_QR_URL with dc_lot_t::text1=URL
- * - DC_QR_ERROR with dc_lot_t::text1=Error string
+ *   the scanned QR code contains a fingerprint but no email address;
+ *   suggest the user to establish an encrypted connection first.
+ *
+ * - DC_QR_ACCOUNT dc_lot_t::text1=domain:
+ *   ask the user if they want to create an account on the given domain,
+ *   if so, call dc_set_config_from_qr() and then dc_configure().
+ *
+ * - DC_QR_WEBRTC_INSTANCE with dc_lot_t::text1=domain:
+ *   ask the user if they want to use the given service for video chats;
+ *   if so, call dc_set_config_from_qr().
+ *
+ * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
+ *   email-address scanned,
+ *   ask the user if they want to start chatting;
+ *   if so, call dc_create_chat_by_contact_id()
+ *
+ * - DC_QR_TEXT with dc_lot_t::text1=Text:
+ *   Text scanned,
+ *   ask the user eg. if they want copy to clipboard.
+ *
+ * - DC_QR_URL with dc_lot_t::text1=URL:
+ *   URL scanned,
+ *   ask the user eg. if they want to open a browser or copy to clipboard.
+ *
+ * - DC_QR_ERROR with dc_lot_t::text1=Error string:
+ *   show the error to the user.
+ *
+ * - DC_QR_WITHDRAW_VERIFYCONTACT:
+ *   ask the user if they want to withdraw the their own qr-code;
+ *   if so, call dc_set_config_from_qr().
+ *
+ * - DC_QR_WITHDRAW_VERIFYGROUP with text1=groupname:
+ *   ask the user if they want to withdraw the group-invite code;
+ *   if so, call dc_set_config_from_qr().
+ *
+ * - DC_QR_REVIVE_VERIFYCONTACT:
+ *   ask the user if they want to revive their withdrawn qr-code;
+ *   if so, call dc_set_config_from_qr().
+ *
+ * - DC_QR_REVIVE_VERIFYGROUP with text1=groupname:
+ *   ask the user if they want to revive the withdrawn group-invite code;
+ *   if so, call dc_set_config_from_qr().
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -2347,7 +2396,7 @@ void dc_str_unref (char* str);
  * The account manager takes an directory
  * where all context-databases are placed in.
  * To add a context to the account manager,
- * use dc_accounts_add_account(), dc_accounts_import_account or dc_accounts_migrate_account().
+ * use dc_accounts_add_account() or dc_accounts_migrate_account().
  * All account information are persisted.
  * To remove a context from the account manager,
  * use dc_accounts_remove_account().
@@ -2392,21 +2441,6 @@ void           dc_accounts_unref                (dc_accounts_t* accounts);
 uint32_t       dc_accounts_add_account          (dc_accounts_t* accounts);
 
 
-/**
- * Import a tarfile-backup to the account manager.
- * On success, a new account is added to the account-manager,
- * with all the data provided by the backup-file.
- * Moreover, the newly created account will be the selected one.
- *
- * @memberof dc_accounts_t
- * @param accounts Account manager as created by dc_accounts_new().
- * @param tarfile Backup as created by dc_imex().
- * @return Account-id, use dc_accounts_get_account() to get the context object.
- *     On errors, 0 is returned.
- */
-uint32_t       dc_accounts_import_account       (dc_accounts_t* accounts, const char* tarfile);
-
-
 /**
  * Migrate independent accounts into accounts managed by the account manager.
  * This will _move_ the database-file and all blob-files to the directory managed
@@ -2476,6 +2510,7 @@ dc_context_t*  dc_accounts_get_account          (dc_accounts_t* accounts, uint32
  *     unmanaged account-context as created by dc_context_new().
  *     Once you do no longer need the context-object, you have to call dc_context_unref() on it,
  *     which, however, will not close the account but only decrease a reference counter.
+ *     If there is no selected account, NULL is returned.
  */
 dc_context_t*  dc_accounts_get_selected_account (dc_accounts_t* accounts);
 
@@ -2491,6 +2526,23 @@ dc_context_t*  dc_accounts_get_selected_account (dc_accounts_t* accounts);
 int            dc_accounts_select_account       (dc_accounts_t* accounts, uint32_t account_id);
 
 
+/**
+ * This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
+ *
+ * iOS can:
+ * - call dc_start_io() (in case IO was not running)
+ * - call dc_maybe_network()
+ * - while dc_accounts_all_work_done() returns false:
+ *   -  Wait for #DC_EVENT_CONNECTIVITY_CHANGED
+ *
+ * @memberof dc_accounts_t
+ * @param accounts Account manager as created by dc_accounts_new().
+ * @return Whether all accounts finished their background work.
+ *      #DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
+ */
+int            dc_accounts_all_work_done        (dc_accounts_t* accounts);
+
+
 /**
  * Start job and IMAP/SMTP tasks for all accounts managed by the account manager.
  * If IO is already running, nothing happens.
@@ -2526,6 +2578,21 @@ void           dc_accounts_stop_io              (dc_accounts_t* accounts);
 void           dc_accounts_maybe_network        (dc_accounts_t* accounts);
 
 
+/**
+ * This function can be called when there is a hint that the network is lost.
+ * This is similar to dc_accounts_maybe_network(), however,
+ * it does not retry job processing.
+ *
+ * dc_accounts_maybe_network_lost() is needed only on systems
+ * where the core cannot find out the connectivity loss on its own, eg. iOS.
+ * The function is not needed on Android, MacOS, Windows or Linux.
+ *
+ * @memberof dc_accounts_t
+ * @param accounts Account manager as created by dc_accounts_new().
+ */
+void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
+
+
 /**
  * Create the event emitter that is used to receive events.
  *
@@ -2745,15 +2812,9 @@ int              dc_array_search_id          (const dc_array_t* array, uint32_t
  * and for each messages that is scrolled into view, dc_get_msg() is called then.
  *
  * Why no listflags?
- * Without listflags, dc_get_chatlist() adds the deaddrop
- * and the archive "link" automatically as needed.
+ * Without listflags, dc_get_chatlist() adds
+ * the archive "link" automatically as needed.
  * The UI can just render these items differently then.
- * Although the deaddrop link is currently always the first entry
- * and only present on new messages,
- * there is the rough idea that it can be optionally always present
- * and sorted into the list by date.
- * Rendering the deaddrop in the described way
- * would not add extra work in the UI then.
  */
 
 
@@ -2894,7 +2955,6 @@ char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat
  */
 
 
-#define         DC_CHAT_ID_DEADDROP          1 // virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 #define         DC_CHAT_ID_TRASH             3 // messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 #define         DC_CHAT_ID_ARCHIVED_LINK     6 // only an indicator in a chatlist
 #define         DC_CHAT_ID_ALLDONE_HINT      7 // only an indicator in a chatlist
@@ -2921,7 +2981,6 @@ void            dc_chat_unref                (dc_chat_t* chat);
  * Get chat ID. The chat ID is the ID under which the chat is filed in the database.
  *
  * Special IDs:
- * - DC_CHAT_ID_DEADDROP         (1) - Virtual chat containing messages which senders are not confirmed by the user.
  * - DC_CHAT_ID_ARCHIVED_LINK    (6) - A link at the end of the chatlist, if present the UI should show the button "Archived chats"-
  *
  * "Normal" chat IDs are larger than these special IDs (larger than DC_CHAT_ID_LAST_SPECIAL).
@@ -2980,7 +3039,7 @@ char*           dc_chat_get_name             (const dc_chat_t* chat);
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return Path and file if the profile image, if any.
+ * @return Path and file of the profile image, if any.
  *     NULL otherwise.
  *     Must be released using dc_str_unref() after usage.
  */
@@ -3013,6 +3072,25 @@ uint32_t        dc_chat_get_color            (const dc_chat_t* chat);
 int             dc_chat_get_visibility       (const dc_chat_t* chat);
 
 
+/**
+ * Check if a chat is a contact request chat.
+ *
+ * UI should display such chats with a [New] badge in the chatlist.
+ *
+ * When such chat is opened, user should be presented with a set of
+ * options instead of the message composition area, for example:
+ * - Accept chat (dc_accept_chat())
+ * - Block chat (dc_block_chat())
+ * - Delete chat (dc_delete_chat())
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is a contact request chat
+ *     0=chat is not a contact request chat
+ */
+int             dc_chat_is_contact_request   (const dc_chat_t* chat);
+
+
 /**
  * Check if a group chat is still unpromoted.
  *
@@ -3065,8 +3143,8 @@ int             dc_chat_is_device_talk       (const dc_chat_t* chat);
 
 
 /**
- * Check if messages can be sent to a give chat.
- * This is not true e.g. for the deaddrop or for the device-talk, cmp. dc_chat_is_device_talk().
+ * Check if messages can be sent to a given chat.
+ * This is not true e.g. for contact requests or for the device-talk, cmp. dc_chat_is_device_talk().
  *
  * Calling dc_send_msg() for these chats will fail
  * and the UI may decide to hide input controls therefore.
@@ -3150,10 +3228,6 @@ int64_t          dc_chat_get_remaining_mute_duration (const dc_chat_t* chat);
 #define         DC_STATE_OUT_MDN_RCVD        28
 
 
-#define         DC_MAX_GET_TEXT_LEN          30000 // approx. max. length returned by dc_msg_get_text()
-#define         DC_MAX_GET_INFO_LEN          100000 // approx. max. length returned by dc_get_msg_info()
-
-
 /**
  * Create new message object. Message objects are needed e.g. for sending messages using
  * dc_send_msg().  Moreover, they are returned e.g. from dc_get_msg(),
@@ -3210,9 +3284,6 @@ uint32_t        dc_msg_get_from_id            (const dc_msg_t* msg);
 /**
  * Get the ID of chat the message belongs to.
  * To get details about the chat, pass the returned ID to dc_get_chat().
- * If a message is still in the deaddrop, the ID DC_CHAT_ID_DEADDROP is returned
- * although internally another ID is used.
- * (to get that internal id, use dc_msg_get_real_chat_id())
  *
  * @memberof dc_msg_t
  * @param msg The message object.
@@ -3221,19 +3292,6 @@ uint32_t        dc_msg_get_from_id            (const dc_msg_t* msg);
 uint32_t        dc_msg_get_chat_id            (const dc_msg_t* msg);
 
 
-/**
- * Get the ID of chat the message belongs to.
- * To get details about the chat, pass the returned ID to dc_get_chat().
- * In contrast to dc_msg_get_chat_id(), this function returns the chat-id also
- * for messages in the deaddrop.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @return The ID of the chat the message belongs to, 0 on errors.
- */
-uint32_t        dc_msg_get_real_chat_id       (const dc_msg_t* msg);
-
-
 /**
  * Get the type of the message.
  *
@@ -3483,6 +3541,16 @@ int             dc_msg_get_duration           (const dc_msg_t* msg);
  */
 int             dc_msg_get_showpadlock        (const dc_msg_t* msg);
 
+/**
+ * Check if incoming message is a bot message, i.e. automatically submitted.
+ *
+ * Return value for outgoing messages is unspecified.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return 1=message is submitted automatically, 0=message is not automatically submitted.
+ */
+int             dc_msg_is_bot                 (const dc_msg_t* msg); 
 
 /**
  * Get ephemeral timer duration for message.
@@ -4892,26 +4960,6 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_ERROR                    400
 
 
-/**
- * An action cannot be performed because there is no network available.
- *
- * The library will typically try over after a some time
- * and when dc_maybe_network() is called.
- *
- * Network errors should be reported to users in a non-disturbing way,
- * however, as network errors may come in a sequence,
- * it is not useful to raise each an every error to the user.
- *
- * Moreover, if the UI detects that the device is offline,
- * it is probably more useful to report this to the user
- * instead of the string from data2.
- *
- * @param data1 0
- * @param data2 (char*) Error string, always set, never NULL.
- */
-#define DC_EVENT_ERROR_NETWORK            401
-
-
 /**
  * An action cannot be performed because the user is not in the group.
  * Reported e.g. after a call to
@@ -5096,6 +5144,18 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
+
+/**
+ * The connectivity to the server changed.
+ * This means that you should refresh the connectivity view
+ * and possibly the connectivtiy HTML; see dc_get_connectivity() and
+ * dc_get_connectivity_html() for details.
+ *
+ * @param data1 0
+ * @param data2 0
+ */
+#define DC_EVENT_CONNECTIVITY_CHANGED             2100
+
 /**
  * @}
  */
@@ -5264,11 +5324,6 @@ void dc_event_unref(dc_event_t* event);
 /// Used in summaries.
 #define DC_STR_VOICEMESSAGE               7
 
-/// "Contact requests"
-///
-/// Used as the name for the corresponding chat.
-#define DC_STR_DEADDROP                   8
-
 /// "Image"
 ///
 /// Used in summaries.
@@ -5423,13 +5478,6 @@ void dc_event_unref(dc_event_t* event);
 /// - %1$s will be replaced by the failing login name
 #define DC_STR_CANNOT_LOGIN               60
 
-/// "Could not connect to %1$s: %2$s"
-///
-/// Used in error strings.
-/// - %1$s will be replaced by the failing server
-/// - %2$s by a the error message as returned from the server
-#define DC_STR_SERVER_RESPONSE            61
-
 /// "%1$s by %2$s"
 ///
 /// Used to concretize actions,
@@ -5625,6 +5673,18 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by the number of weeks (always >1) the timer is set to.
 #define DC_STR_EPHEMERAL_WEEKS            96
 
+/// "Forwarded"
+///
+/// Used in message summary text for notifications and chatlist.
+#define DC_STR_FORWARDED                  97
+
+/// "Quota exceeding, already %1$s%% used."
+///
+/// Used as device message text.
+///
+/// `%1$s` will be replaced by the percentage used
+#define DC_STR_QUOTA_EXCEEDING_MSG_BODY   98
+
 /**
  * @}
  */

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir=${{prefix}}/lib
-includedir=${{prefix}}/include
+libdir={libdir}
+includedir={includedir}
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/string.rs

@@ -17,15 +17,12 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char;
-    if !s.is_null() {
-        ret = libc::strdup(s);
-        assert!(!ret.is_null());
+    let ret: *mut libc::c_char = if !s.is_null() {
+        libc::strdup(s)
     } else {
-        ret = libc::calloc(1, 1) as *mut libc::c_char;
-        assert!(!ret.is_null());
-    }
-
+        libc::calloc(1, 1) as *mut libc::c_char
+    };
+    assert!(!ret.is_null());
     ret
 }
 
@@ -170,15 +167,20 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl<T: AsRef<str>> Strdup for T {
+impl Strdup for str {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self.as_ref());
+        let tmp = CString::new_lossy(self);
         dc_strdup(tmp.as_ptr())
     }
 }
 
-// We can not implement for AsRef<OsStr> because we already implement
-// AsRev<str> and this conflicts.  So implement for Path directly.
+impl Strdup for String {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        let s: &str = self;
+        s.strdup()
+    }
+}
+
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -186,6 +188,13 @@ impl Strdup for std::path::Path {
     }
 }
 
+impl Strdup for [u8] {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        let tmp = CString::new_lossy(self);
+        dc_strdup(tmp.as_ptr())
+    }
+}
+
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name

deltachat_derive/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "deltachat_derive"
+version = "2.0.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+edition = "2018"
+license = "MPL-2.0"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "1.0.74"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -0,0 +1,47 @@
+#![recursion_limit = "128"]
+extern crate proc_macro;
+
+use crate::proc_macro::TokenStream;
+use quote::quote;
+
+// For now, assume (not check) that these macroses are applied to enum without
+// data.  If this assumption is violated, compiler error will point to
+// generated code, which is not very user-friendly.
+
+#[proc_macro_derive(ToSql)]
+pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(input).unwrap();
+    let name = &ast.ident;
+
+    let gen = quote! {
+        impl rusqlite::types::ToSql for #name {
+            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+                let num = *self as i64;
+                let value = rusqlite::types::Value::Integer(num);
+                let output = rusqlite::types::ToSqlOutput::Owned(value);
+                std::result::Result::Ok(output)
+            }
+        }
+    };
+    gen.into()
+}
+
+#[proc_macro_derive(FromSql)]
+pub fn from_sql_derive(input: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(input).unwrap();
+    let name = &ast.ident;
+
+    let gen = quote! {
+        impl rusqlite::types::FromSql for #name {
+            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+                let inner = rusqlite::types::FromSql::column_result(col)?;
+                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
+                    Ok(value)
+                } else {
+                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
+                }
+            }
+        }
+    };
+    gen.into()
+}

examples/repl/cmdline.rs

@@ -17,7 +17,7 @@ use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
 use deltachat::lot::LotState;
-use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
@@ -34,7 +34,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 1 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM jobs;"))
+            .execute("DELETE FROM jobs;", paramsv![])
             .await
             .unwrap();
         println!("(1) Jobs reset.");
@@ -42,7 +42,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 2 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM acpeerstates;"))
+            .execute("DELETE FROM acpeerstates;", paramsv![])
             .await
             .unwrap();
         println!("(2) Peerstates reset.");
@@ -50,7 +50,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 4 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM keypairs;"))
+            .execute("DELETE FROM keypairs;", paramsv![])
             .await
             .unwrap();
         println!("(4) Private keypairs reset.");
@@ -58,34 +58,35 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 8 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM contacts WHERE id>9;"))
+            .execute("DELETE FROM contacts WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM chats WHERE id>9;"))
+            .execute("DELETE FROM chats WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM chats_contacts;"))
+            .execute("DELETE FROM chats_contacts;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM msgs WHERE id>9;"))
+            .execute("DELETE FROM msgs WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query(
+            .execute(
                 "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-            ))
+                paramsv![],
+            )
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM leftgrps;"))
+            .execute("DELETE FROM leftgrps;", paramsv![])
             .await
             .unwrap();
         println!("(8) Rest but server config reset.");
@@ -350,6 +351,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
+                 connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -371,6 +373,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  getlocations [<contact-id>]\n\
                  send <text>\n\
                  sendimage <file> [<text>]\n\
+                 sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
@@ -386,10 +389,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
-                 ===========================Contact requests==\n\
-                 decidestartchat <msg-id>\n\
-                 decideblock <msg-id>\n\
-                 decidenotnow <msg-id>\n\
+                 accept <chat-id>\n\
+                 decline <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
@@ -456,20 +457,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, &dir).await?;
+            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, arg1).await?;
+            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -509,6 +510,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "info" => {
             println!("{:#?}", context.get_info().await);
         }
+        "connectivity" => {
+            let file = dirs::home_dir()
+                .unwrap_or_default()
+                .join("connectivity.html");
+            match context.get_connectivity_html().await {
+                Ok(html) => {
+                    fs::write(&file, html)?;
+                    println!("Report written to: {:#?}", file);
+                }
+                Err(err) => {
+                    bail!("Failed to get connectivity html: {}", err);
+                }
+            }
+        }
         "maybenetwork" => {
             context.maybe_network().await;
         }
@@ -536,7 +551,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}{}",
+                        "{}#{}: {} [{} fresh] {}{}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -548,8 +563,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Pinned => "📌",
                         },
                         if chat.is_protected() { "🛡️" } else { "" },
+                        if chat.is_contact_request() {
+                            "🆕"
+                        } else {
+                            ""
+                        },
                     );
-                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await?;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
@@ -602,7 +622,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let time_start = std::time::SystemTime::now();
+            let msglist =
+                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
+
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -657,44 +681,23 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "{} messages.",
                 sel_chat.get_id().get_msg_cnt(&context).await?
             );
+
+            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
+            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
+
+            println!(
+                "{:?} to create this list, {:?} to mark all messages as noticed.",
+                time_needed, time_noticed_needed
+            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id: u32 = arg1.parse()?;
-            let chat_id = chat::create_by_contact_id(&context, contact_id).await?;
+            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
-        "decidestartchat" | "createchatbymsg" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id = MsgId::new(arg1.parse()?);
-            match message::decide_on_contact_request(
-                &context,
-                msg_id,
-                ContactRequestDecision::StartChat,
-            )
-            .await
-            {
-                Some(chat_id) => {
-                    let chat = Chat::load_from_db(&context, chat_id).await?;
-                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
-                }
-                None => println!("Cannot crate chat."),
-            }
-        }
-        "decidenotnow" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id = MsgId::new(arg1.parse()?);
-            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
-                .await;
-        }
-        "decideblock" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id = MsgId::new(arg1.parse()?);
-            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
-                .await;
-        }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -852,12 +855,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), "".into()).await?;
         }
-        "sendimage" | "sendfile" => {
+        "sendimage" | "sendsticker" | "sendfile" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No file given.");
 
             let mut msg = Message::new(if arg0 == "sendimage" {
                 Viewtype::Image
+            } else if arg0 == "sendsticker" {
+                Viewtype::Sticker
             } else {
                 Viewtype::File
             });
@@ -1009,6 +1014,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
+        "accept" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.accept(&context).await?;
+        }
+        "blockchat" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.block(&context).await?;
+        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1046,7 +1061,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await;
+            message::markseen_msgs(&context, msg_ids).await?;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -1073,7 +1088,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, book).await?;
+                Contact::add_address_book(&context, &book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }
@@ -1121,12 +1136,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id = arg1.parse()?;
-            Contact::block(&context, contact_id).await;
+            Contact::block(&context, contact_id).await?;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id = arg1.parse()?;
-            Contact::unblock(&context, contact_id).await;
+            Contact::unblock(&context, contact_id).await?;
         }
         "listblocked" => {
             let contacts = Contact::get_all_blocked(&context).await?;
@@ -1153,7 +1168,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1).await {
+            let socks5_enabled = context
+                .get_config_bool(config::Config::Socks5Enabled)
+                .await?;
+            match provider::get_provider_info(arg1, socks5_enabled).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -26,8 +26,9 @@ use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
+use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
 
 mod cmdline;
@@ -56,9 +57,6 @@ fn receive_event(event: EventType) {
         EventType::Error(msg) => {
             error!("{}", msg);
         }
-        EventType::ErrorNetwork(msg) => {
-            error!("[NETWORK] msg={}", msg);
-        }
         EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
@@ -156,7 +154,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 9] = [
+const DB_COMMANDS: [&str; 10] = [
     "info",
     "set",
     "get",
@@ -164,18 +162,16 @@ const DB_COMMANDS: [&str; 9] = [
     "configure",
     "connect",
     "disconnect",
+    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 34] = [
+const CHAT_COMMANDS: [&str; 33] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
-    "decidestartchat",
-    "decideblock",
-    "decidenotnow",
     "creategroup",
     "createverified",
     "addmember",
@@ -203,6 +199,8 @@ const CHAT_COMMANDS: [&str; 34] = [
     "protect",
     "unprotect",
     "delchat",
+    "accept",
+    "blockchat",
 ];
 const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
@@ -237,7 +235,9 @@ const MISC_COMMANDS: [&str; 10] = [
 ];
 
 impl Hinter for DcHelper {
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
+    type Hint = String;
+
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -259,11 +259,10 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
-        if prompt == PROMPT {
+    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
+        if default {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -284,6 +283,7 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
+impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
@@ -317,8 +317,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
@@ -392,7 +392,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {

examples/simple.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use deltachat::chat;
+use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
@@ -19,7 +19,7 @@ fn cb(event: EventType) {
         EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
+        EventType::Error(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
+    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

python/README.rst

@@ -58,12 +58,13 @@ end-to-end tests that require accounts on real e-mail servers.
 running "live" tests with temporary accounts
 ---------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
 
-    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
 
-With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
-These accounts exists for one 1hour and then are removed completely.
+    export DCC_NEW_TMP_EMAIL=<URL you got from us>
+
+With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
 One hour is enough to invoke pytest and run all offline and online tests:
 
     pytest

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -9,8 +9,6 @@ import subprocess
 import tempfile
 import textwrap
 import types
-from os.path import abspath
-from os.path import dirname as dn
 
 import cffi
 
@@ -50,6 +48,7 @@ def system_build_flags():
     flags.objs = []
     flags.incs = []
     flags.extra_link_args = []
+    return flags
 
 
 def extract_functions(flags):
@@ -151,6 +150,7 @@ def extract_defines(flags):
                 | DC_PROVIDER
                 | DC_KEY_GEN
                 | DC_IMEX
+                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -168,11 +168,8 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
+        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()

python/src/deltachat/account.py

@@ -330,9 +330,6 @@ class Account(object):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def _create_chat_by_message_id(self, msg_id):
-        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
-
     def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
@@ -367,9 +364,6 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_deaddrop_chat(self):
-        return Chat(self, const.DC_CHAT_ID_DEADDROP)
-
     def get_device_chat(self):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
@@ -574,6 +568,15 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
+    def get_connectivity(self):
+        return lib.dc_get_connectivity(self._dc_context)
+
+    def get_connectivity_html(self):
+        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
+
+    def all_work_done(self):
+        return lib.dc_all_work_done(self._dc_context)
+
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 

python/src/deltachat/chat.py

@@ -50,6 +50,14 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
+    def block(self):
+        """Block this chat."""
+        lib.dc_block_chat(self.account._dc_context, self.id)
+
+    def accept(self):
+        """Accept this contact request chat."""
+        lib.dc_accept_chat(self.account._dc_context, self.id)
+
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self):
@@ -59,13 +67,6 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_deaddrop(self):
-        """ return true if this chat is a deaddrop chat.
-
-        :returns: True if chat is the deaddrop chat, False otherwise.
-        """
-        return self.id == const.DC_CHAT_ID_DEADDROP
-
     def is_muted(self):
         """ return true if this chat is muted.
 
@@ -73,6 +74,13 @@ class Chat(object):
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
+    def is_contact_request(self):
+        """ return True if this chat is a contact request chat.
+
+        :returns: True if chat is a contact request chat, False otherwise.
+        """
+        return lib.dc_chat_is_contact_request(self._dc_chat)
+
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -84,7 +92,7 @@ class Chat(object):
 
     def can_send(self):
         """Check if messages can be sent to a give chat.
-        This is not true eg. for the deaddrop or for the device-talk
+        This is not true eg. for the contact requests or for the device-talk
 
         :returns: True if the chat is writable, False otherwise
         """

python/src/deltachat/direct_imap.py

@@ -251,7 +251,16 @@ class DirectImap:
             return res
 
     def append(self, folder, msg):
+        """Upload a message to *folder*.
+        Trailing whitespace or a linebreak at the beginning will be removed automatically.
+        """
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
         self.conn.append(folder, msg)
+
+    def get_uid_by_message_id(self, message_id):
+        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
+        if len(msgs) == 0:
+            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
+        return msgs[0]

python/src/deltachat/events.py

@@ -111,6 +111,33 @@ class FFIEventTracker:
             if m is not None:
                 return m.groups()
 
+    def wait_for_connectivity(self, connectivity):
+        """Wait for the specified connectivity.
+        This only works reliably if the connectivity doesn't change
+        again too quickly, otherwise we might miss it."""
+        while 1:
+            if self.account.get_connectivity() == connectivity:
+                return
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+    def wait_for_connectivity_change(self, previous, expected_next):
+        """Wait until the connectivity changes to `expected_next`.
+        Fails the test if it changes to something else."""
+        while 1:
+            current = self.account.get_connectivity()
+            if current == expected_next:
+                return
+            elif current != previous:
+                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
+
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+    def wait_for_all_work_done(self):
+        while 1:
+            if self.account.all_work_done():
+                return
+            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (to deaddrop or chat). """
+        """ Called on any incoming message (both existing chats and contact requests). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):

python/src/deltachat/message.py

@@ -61,16 +61,13 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a deaddrop contact request
+        If the message is a contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        from .chat import Chat
-        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
-        ctx = self.account._dc_context
-        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
-        return Chat(self.account, chat_id)
+        self.chat.accept()
+        return self.chat
 
     @props.with_doc
     def id(self):
@@ -141,6 +138,10 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
+    def is_bot(self):
+        """ return True if this message is submitted automatically. """
+        return bool(lib.dc_msg_is_bot(self._dc_msg))
+
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))

python/tests/auditwheels.py

@@ -1,15 +1,17 @@
 
 import os
-import sys
+import platform
 import subprocess
+import sys
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
+    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_x86_64"])
+                 "--plat", "manylinux2014_" + arch])

python/tests/test_account.py

@@ -909,12 +909,11 @@ class TestOnlineAccount:
         msg_in = ac2.get_message_by_id(msg_out.id)
         assert msg_in.text == "message2"
 
-        lp.sec("ac2: check that the message arrive in deaddrop")
+        lp.sec("ac2: check that the message arrived in a chat")
         chat2 = msg_in.chat
         assert msg_in in chat2.get_messages()
         assert not msg_in.is_forwarded()
-        assert chat2.is_deaddrop()
-        assert chat2 == ac2.get_deaddrop_chat()
+        assert chat2.is_contact_request()
 
         lp.sec("ac2: create new chat and forward message to it")
         chat3 = ac2.create_group_chat("newgroup")
@@ -979,16 +978,16 @@ class TestOnlineAccount:
         assert not msg2.is_forwarded()
         assert msg2.get_sender_contact().display_name == ac1.get_config("displayname")
 
-        lp.sec("check the message arrived in contact-requests/deaddrop")
+        lp.sec("check the message arrived in contact request chat")
         chat2 = msg2.chat
         assert msg2 in chat2.get_messages()
-        assert chat2.is_deaddrop()
-        assert chat2.count_fresh_messages() == 0
+        assert chat2.is_contact_request()
+        assert chat2.count_fresh_messages() == 1
         assert msg2.time_received >= msg1.time_sent
 
         lp.sec("create new chat with contact and verify it's proper")
         chat2b = msg2.create_chat()
-        assert not chat2b.is_deaddrop()
+        assert not chat2b.is_contact_request()
         assert chat2b.count_fresh_messages() == 1
 
         lp.sec("mark chat as noticed")
@@ -1033,6 +1032,33 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_moved_markseen(self, acfactory, lp):
+        """Test that message already moved to DeltaChat folder is marked as seen."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
+        ac2 = acfactory.get_online_configuring_account()
+        acfactory.wait_configure_and_start_io([ac1, ac2])
+        ac1.set_config("bcc_self", "1")
+
+        ac1.direct_imap.idle_start()
+        ac1.create_chat(ac2).send_text("Hello!")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.stop_io()
+
+        # Emulate moving of the message to DeltaChat folder by Sieve rule.
+        # mailcow server contains this rule by default.
+        ac1.direct_imap.conn.move(["*"], "DeltaChat")
+
+        ac1.direct_imap.select_folder("DeltaChat")
+        ac1.direct_imap.idle_start()
+        ac1.start_io()
+        ac1.direct_imap.idle_wait_for_seen()
+        ac1.direct_imap.idle_done()
+
+        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
+        flags = fetch[-1][b'FLAGS']
+        is_seen = b'\\Seen' in flags
+        assert is_seen
+
     def test_message_override_sender_name(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = acfactory.get_accepted_chat(ac1, ac2)
@@ -1071,10 +1097,10 @@ class TestOnlineAccount:
         # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
         ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
         ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
-        acfactory.wait_configure_and_start_io()
 
-        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
-        msg = ac2._evtracker.wait_next_incoming_message()
+        acfactory.wait_configure_and_start_io()
+        # Do not send BCC to self, we only want to test MDN on ac1.
+        ac1.set_config("bcc_self", "0")
 
         folder = "mvbox" if mvbox_move else "inbox"
         ac1.direct_imap.select_config_folder(folder)
@@ -1082,6 +1108,9 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.direct_imap.idle_start()
 
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
         ac2.mark_seen_messages([msg])
 
         ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
@@ -1289,9 +1318,11 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
-    def test_dont_show_emails_in_draft_folder(self, acfactory):
+    def test_dont_show_emails_in_draft_folder(self, acfactory, lp):
         """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
-        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email."""
+        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email.
+
+        If the draft email is sent out later (i.e. moved to "Sent"), it must be shown."""
         ac1 = acfactory.get_online_configuring_account()
         ac1.set_config("show_emails", "2")
         ac1.create_contact("alice@example.com").create_chat()
@@ -1312,7 +1343,7 @@ class TestOnlineAccount:
             Message-ID: <aepiors@example.org>
             Content-Type: text/plain; charset=utf-8
 
-            message in Drafts
+            message in Drafts that is moved to Sent later
         """.format(ac1.get_config("configured_addr")))
         ac1.direct_imap.append("Sent", """
             From: ac1 <{}>
@@ -1325,6 +1356,7 @@ class TestOnlineAccount:
         """.format(ac1.get_config("configured_addr")))
 
         ac1.set_config("scan_all_folders_debounce_secs", "0")
+        lp.sec("All prepared, now let DC find the message")
         ac1.start_io()
 
         msg = ac1._evtracker.wait_next_messages_changed()
@@ -1335,6 +1367,18 @@ class TestOnlineAccount:
         assert msg.text == "subj – message in Sent"
         assert len(msg.chat.get_messages()) == 1
 
+        ac1.stop_io()
+        lp.sec("'Send out' the draft, i.e. move it to the Sent folder, and wait for DC to display it this time")
+        ac1.direct_imap.select_folder("Drafts")
+        uid = ac1.direct_imap.get_uid_by_message_id("aepiors@example.org")
+        ac1.direct_imap.conn.move(uid, "Sent")
+
+        ac1.start_io()
+        msg2 = ac1._evtracker.wait_next_messages_changed()
+
+        assert msg2.text == "subj – message in Drafts that is moved to Sent later"
+        assert len(msg.chat.get_messages()) == 2
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1385,6 +1429,33 @@ class TestOnlineAccount:
         # Majority prefers encryption now
         assert msg5.is_encrypted()
 
+    def test_bot(self, acfactory, lp):
+        """Test that bot messages can be identified as such"""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("bot", "0")
+        ac2.set_config("bot", "1")
+
+        lp.sec("ac1: create chat with ac2")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+
+        lp.sec("sending a message from ac1 to ac2")
+        text1 = "hello"
+        chat.send_text(text1)
+
+        lp.sec("wait for ac2 to receive a message")
+        msg_in = ac2._evtracker.wait_next_incoming_message()
+        assert msg_in.text == text1
+        assert not msg_in.is_bot()
+
+        lp.sec("sending a message from ac2 to ac1")
+        text2 = "reply"
+        msg_in.chat.send_text(text2)
+
+        lp.sec("wait for ac1 to receive a message")
+        msg_in = ac1._evtracker.wait_next_incoming_message()
+        assert msg_in.text == text2
+        assert msg_in.is_bot()
+
     def test_quote_encrypted(self, acfactory, lp):
         """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1713,7 +1784,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
         assert chat1.is_protected()
@@ -1744,6 +1815,29 @@ class TestOnlineAccount:
         assert msg.text == "world"
         assert msg.is_encrypted()
 
+        lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
+        qr = ac1.get_setup_contact_qr()
+
+        lp.sec("ac3: start QR-code based setup contact protocol")
+        ch = ac3.qr_setup_contact(qr)
+        assert ch.id >= 10
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+        lp.sec("ac1: add ac3 to verified group")
+        chat1.add_contact(ac3)
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.is_system_message()
+        assert not msg.error
+
+        lp.sec("ac2: send message and let ac3 read it")
+        chat2.send_text("hi")
+        # Skip system message about added member
+        ac3._evtracker.wait_next_incoming_message()
+        msg = ac3._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.is_encrypted()
+
     def test_set_get_contact_avatar(self, acfactory, data, lp):
         lp.sec("configuring ac1 and ac2")
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1758,7 +1852,7 @@ class TestOnlineAccount:
 
         lp.sec("ac2: wait for receiving message and avatar from ac1")
         msg2 = ac2._evtracker.wait_next_messages_changed()
-        assert msg2.chat.is_deaddrop()
+        assert msg2.chat.is_contact_request()
         received_path = msg2.get_sender_contact().get_profile_image()
         assert open(received_path, "rb").read() == open(p, "rb").read()
 
@@ -1833,6 +1927,8 @@ class TestOnlineAccount:
         ev = in_list.get(timeout=10)
         assert ev.action == "chat-modified"
         ev = in_list.get(timeout=10)
+        assert ev.action == "chat-modified"
+        ev = in_list.get(timeout=10)
         assert ev.action == "added"
         assert ev.message.get_sender_contact().addr == ac1_addr
         assert ev.contact.addr == "devnull@testrun.org"
@@ -1947,6 +2043,125 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
+    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
+    def test_connectivity(self, acfactory, lp, inbox_watch):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1.set_config("inbox_watch", inbox_watch)
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
+
+        lp.sec("Test stop_io() and start_io()")
+        ac1.stop_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+        ac1.start_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTING, const.DC_CONNECTIVITY_CONNECTED)
+
+        lp.sec("Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " +
+               "all messages are fetched")
+
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        ac2.create_chat(ac1).send_text("Hi")
+
+        ac1.direct_imap.idle_check(terminate=False)
+        ac1.maybe_network()
+
+        ac1._evtracker.wait_for_all_work_done()
+        msgs = ac1.create_chat(ac2).get_messages()
+        assert len(msgs) == 1
+        assert msgs[0].text == "Hi"
+
+        lp.sec("Test that the connectivity changes to WORKING while new messages are fetched")
+
+        ac2.create_chat(ac1).send_text("Hi 2")
+
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.maybe_network()
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
+        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
+
+        msgs = ac1.create_chat(ac2).get_messages()
+        assert len(msgs) == 2
+        assert msgs[1].text == "Hi 2"
+
+        lp.sec("Test that the connectivity doesn't flicker to WORKING if there are no new messages")
+
+        ac1.maybe_network()
+        while 1:
+            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
+            if ac1.all_work_done():
+                break
+            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+        lp.sec("Test that the connectivity doesn't flicker to WORKING if the sender of the message is blocked")
+        ac1.create_contact(ac2).block()
+
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        ac2.create_chat(ac1).send_text("Hi")
+
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.maybe_network()
+
+        while 1:
+            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
+            if ac1.all_work_done():
+                break
+            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
+
+        lp.sec("Test that the connectivity is NOT_CONNECTED if the password is wrong")
+
+        ac1.set_config("configured_mail_pw", "abc")
+        ac1.stop_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+        ac1.start_io()
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
+        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+    def test_fetch_deleted_msg(self, acfactory, lp):
+        """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
+        hundreds of times, because uid_next was not updated.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2429.
+        """
+        ac1 = acfactory.get_one_online_account()
+        ac1.stop_io()
+
+        ac1.direct_imap.append("INBOX", """
+            From: alice <alice@example.org>
+            Subject: subj
+            To: bob@example.com
+            Chat-Version: 1.0
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Deleted message
+        """)
+        ac1.direct_imap.delete("1:*", expunge=False)
+        ac1.start_io()
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                break
+
+        # The message was downloaded once, now check that it's not downloaded again
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                pytest.fail("The same email was read twice")
+
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "INBOX: Idle entering wait-on-remote state" in ev.data2:
+                break  # DC is done with reading messages
+
     def test_send_receive_locations(self, acfactory, lp):
         now = datetime.utcnow()
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -2332,26 +2547,31 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
-    @pytest.mark.parametrize("folder,move,expected_destination,", [
-        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
-        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
-        ("Spam", False, "INBOX")  # ...emails are moved from the spam folder to the Inbox
+    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
+        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
+        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
     ])
     # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
     # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
         """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
         variant = folder + "-" + str(move) + "-" + expected_destination
         lp.sec("Testing variant " + variant)
         ac1 = acfactory.get_online_configuring_account(move=move)
         ac2 = acfactory.get_online_configuring_account()
+        ac1.set_config("inbox_watch", inbox_watch)
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder(folder)
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing:
-        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.stop_io()
 
         # Send a message to ac1 and move it to the mvbox:
@@ -2368,7 +2588,10 @@ class TestOnlineAccount:
         assert msg.text == "hello"
 
         # Wait until the message was moved (if at all) and we are IDLEing again:
-        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.direct_imap.select_folder(expected_destination)
         assert len(ac1.direct_imap.get_all_messages()) == 1
         if folder != expected_destination:
@@ -2614,7 +2837,6 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2622,7 +2844,6 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2630,4 +2851,3 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
-        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")

python/tox.ini

@@ -1,7 +1,6 @@
 [tox]
-# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py37
+    py3
     lint
     auditwheels
 
@@ -46,10 +45,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    # With Python 3.7 and Sphinx 3.5.0, it throws an exception.
-    # Pin the version to the working one.
+    # Pin dependencies to the versions which actually work with Python 3.5.
     sphinx==3.4.3
-    breathe
+    breathe==4.28.0
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

rust-toolchain

@@ -1 +1 @@
-1.50.0
+1.54.0

scripts/README.md

@@ -1,16 +1,13 @@
 # Continuous Integration Scripts for Delta Chat
 
 Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions),
-[CircleCI](https://app.circleci.com/) and an own build machine.
+Actions](https://docs.github.com/actions)
+and an own build machine.
 
 ## Description of scripts 
 
 - `../.github/workflows` contains jobs run by GitHub Actions.
 
-- `../.circleci/config.yml` describing the build jobs that are run
-  by CircleCI.
-
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
 
@@ -48,3 +45,7 @@ to avoid the relatively large download::
     cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
+
+Additionally, you can install qemu and build arm64 docker image:
+    apt-get install qemu binfmt-support qemu-user-static
+    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

scripts/ci_upload.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-if [ -z "$DEVPI_LOGIN" ] ; then 
-    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
-    exit 0
-fi
-
-set -xe
-
-PYDOCDIR=${1:?directory with python docs}
-WHEELHOUSEDIR=${2:?directory with pre-built wheels}
-DOXYDOCDIR=${3:?directory where doxygen docs to be found}
-SSHTARGET=ci@b1.delta.chat
-
-
-# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
-export BRANCH=${CIRCLE_BRANCH:master}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
-
-
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
-
-# C docs to c.delta.chat
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$DOXYDOCDIR/html/" \
-  delta@c.delta.chat:build-c/${BRANCH}
-
-echo -----------------------
-echo upload wheels 
-echo -----------------------
-
-# Bundle external shared libraries into the wheels
-
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  $WHEELHOUSEDIR \
-  $SSHTARGET:$BUILDDIR 
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    # we rely on the "venv" virtualenv on the remote account to exist 
-    source venv/bin/activate
-    cd $BUILDDIR
-
-    devpi use https://m.devpi.net
-    devpi login dc --password $DEVPI_LOGIN
-
-    N_BRANCH=${BRANCH//[\/]}
-
-    devpi use dc/\$N_BRANCH || {
-        devpi index -c \$N_BRANCH 
-        devpi use dc/\$N_BRANCH
-    }
-    devpi index \$N_BRANCH bases=/root/pypi
-    devpi upload wheelhouse/deltachat*
-
-    # remove devpi non-master dc indices if thy are too old
-    # this script was copied above 
-    python cleanup_devpi_indices.py
-_HERE

scripts/concourse/README.md

@@ -0,0 +1,21 @@
+# Concourse CI pipeline
+
+`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
+that builds C documentation, Python documentation, Python wheels for `x86_64`
+and `aarch64` and Python source packages, and uploads them.
+
+To setup the pipeline run
+```
+fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
+```
+where `secret.yml` contains the following secrets:
+```
+c.delta.chat:
+  private_key: |
+    -----BEGIN RSA PRIVATE KEY-----
+    ...
+    -----END RSA PRIVATE KEY-----
+devpi:
+  login: dc
+  password: ...
+```

scripts/concourse/docs_wheels.yml

@@ -0,0 +1,232 @@
+resources:
+- name: deltachat-core-rust
+  type: git
+  icon: github
+  source:
+    branch: master
+    uri: https://github.com/deltachat/deltachat-core-rust.git
+
+- name: deltachat-core-rust-release
+  type: git
+  icon: github
+  source:
+    branch: master
+    uri: https://github.com/deltachat/deltachat-core-rust.git
+    tag_filter: "py-*"
+
+jobs:
+- name: doxygen
+  plan:
+  - get: deltachat-core-rust
+    trigger: true
+
+  # Build Doxygen documentation
+  - task: build-doxygen
+    config:
+      inputs:
+        - name: deltachat-core-rust
+      outputs:
+        - name: c-docs
+      image_resource:
+        source:
+          repository: hrektts/doxygen
+        type: registry-image
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          cd deltachat-core-rust
+          bash scripts/run-doxygen.sh
+          cd ..
+          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
+
+  - task: upload-c-docs
+    config:
+      inputs:
+        - name: c-docs
+      image_resource:
+        type: registry-image
+        source:
+          repository: alpine
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apk add --no-cache rsync openssh-client
+          mkdir -p ~/.ssh
+          chmod 700 ~/.ssh
+          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
+
+- name: python-x86_64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: vito/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
+        UNPACK_ROOTFS: "true"
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-docs
+          path: ./python/doc/_build/
+        # Source packages
+        - name: py-dist
+          path: ./python/.docker-tox/dist/
+        # Binary wheels
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload python docs to py.delta.chat
+  - task: upload-py-docs
+    config:
+      inputs:
+        - name: py-docs
+      image_resource:
+        type: registry-image
+        source:
+          repository: alpine
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apk add --no-cache rsync openssh-client
+          mkdir -p ~/.ssh
+          chmod 700 ~/.ssh
+          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
+
+  # Upload x86_64 wheels and source packages
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+        - name: py-dist
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*manylinux201*
+          devpi upload py-dist/*
+
+- name: python-aarch64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: vito/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
+        UNPACK_ROOTFS: "true"
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload aarch64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*manylinux201*

scripts/coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-if ! which grcov 2>/dev/null 1>&2; then
+if ! command -v grcov >/dev/null; then
     echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
     echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
     exit 1

scripts/docker-coredeps-arm64/Dockerfile

@@ -0,0 +1,21 @@
+FROM quay.io/pypa/manylinux2014_aarch64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+# Install a recent Perl, needed to install the openssl crate
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_openssl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.34.0
+# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
+
+./Configure -de
+make
+make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.6 environment as the base environment 
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+
+popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+RUST_VERSION=1.54.0
+
+curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
+cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+rustc --version
+cd ..
+rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2010_x86_64
+FROM quay.io/pypa/manylinux2014_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate 
+# Install a recent Perl, needed to install the openssl crate
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.30.0
-# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+PERL_VERSION=5.34.0
+# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

scripts/docker-coredeps/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.5 environment as the base environment 
-/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+# we use the python3.6 environment as the base environment
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.5.*/bin/tox
-ln -s /opt/_internal/cpython-3.5.*/bin/devpi
-ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
 
 popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,16 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
-export PATH=/root/.cargo/bin:$PATH
-rustc --version
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+RUST_VERSION=1.54.0
 
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share
+curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
+cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+rustc --version
+cd ..
+rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/manual_remote_tests.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
-
 set -xe
-export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
-export CIRCLE_BUILD_NUM=$USER
-export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
-export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
 
-time bash scripts/$CIRCLE_JOB.sh 
+JOB=${1:?need to specify 'rust' or 'python'}
+BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
+REPONAME="$(basename $(git rev-parse --show-toplevel))"
+
+time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"

scripts/old/gh-actions-rust.yml

@@ -1,77 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-  push:
-
-env:
-  RUSTFLAGS: -Dwarnings
-
-jobs:
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly]
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      if: matrix.rust == 'nightly'
-      with:
-        command:  check
-        args: --all --bins --examples --tests
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
-
-    - name: tests ignored
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all --release -- --ignored
-
-  check_fmt:
-    name: Checking fmt and docs
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - uses: actions-rs/toolchain@v1
-      with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-
-    - name: fmt
-      run: cargo fmt --all -- --check
-
-#  clippy_check:
-#    name: Clippy check
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - uses: actions/checkout@v1
-#    - uses: actions-rs/toolchain@v1
-#      with:
-#          profile: minimal
-#          toolchain: nightly
-#          override: true
-#          components: clippy
-#
-#    - name: clippy
-#      run: cargo clippy --all

scripts/old/run-python.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library typically run in a docker
-# container that contains all library deps but should also work
-# outside if you have the dependencies installed on your system.
-
-set -e -x
-
-# Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
-
-# install core lib
-
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
-
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
-
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-if [ -n "$TESTS" ]; then
-
-    pushd python
-    # prepare a clean tox run
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-
-    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
-    # allows running of "liveconfig" tests but for speed reasons
-    # we run them only for the highest python version we support
-    
-    # we split out qr-tests run to minimize likelyness of flaky tests
-    # (some qr tests are pretty heavy in terms of send/received
-    # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
-    unset DCC_NEW_TMP_EMAIL
-    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
-    tox --workdir "$TOXWORKDIR" -e auditwheels
-    popd
-fi
-
-
-# if [ -n "$DOCS" ]; then
-#     echo -----------------------
-#     echo generating python docs
-#     echo -----------------------
-#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-# fi

scripts/remote_python_packaging.sh

@@ -1,49 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-set -xe
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-# we seem to need .git for setuptools_scm versioning 
-find .git >>.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-set +x
-
-# we have to create a remote file for the remote-docker run 
-# so we can do a simple ssh command with a TTY
-# so that when our job dies, all container-runs are aborted.
-# sidenote: the circle-ci machinery will kill ongoing jobs
-# if there are new commits and we want to ensure that 
-# everything is terminated/cleaned up and we have no orphaned
-# useless still-running docker-containers consuming resources. 
-
-ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
-    set +x -e
-    shopt -s huponexit
-    cd $BUILDDIR
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
-    set -x
-
-    # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
-
-_HERE
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
-mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

scripts/remote_tests_python.sh

@@ -1,10 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILD_ID=${1:?specify build ID}
 
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -18,7 +17,7 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running $CIRCLE_JOB remotely"
+echo "--- Running Python tests remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e

scripts/remote_tests_rust.sh

@@ -1,20 +1,19 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILD_ID=${1:?specify build ID}
 
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
 
 set -e
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist
 rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
-echo "--- Running $CIRCLE_JOB remotely"
+echo "--- Running Rust tests remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e

scripts/run_all.sh

@@ -19,14 +19,17 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PATH=$PATH:/opt/python/cp36-cp36m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.5
-ln -s /opt/python/cp35-cp35m/bin/python3.5
+rm -f python3.6
 ln -s /opt/python/cp36-cp36m/bin/python3.6
+rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
+rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
+rm -f python3.9
+ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -40,7 +43,7 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
 popd
 
 

scripts/set_core_version.py

@@ -82,9 +82,10 @@ def main():
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit make sure to: ")
+    print("after commit, on master make sure to: ")
     print("")
-    print("   git tag {}".format(newversion))
+    print("   git tag -a {}".format(newversion))
+    print("   git push origin {}".format(newversion))
     print("")
 
 

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.32.0
+Version: 0.33.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -301,9 +301,9 @@ to add a `Chat-Group-Avatar` only on image changes.
 
 A user MAY have a profile-image that MAY be distributed to their contacts.
 To change or set the profile-image,
-the messenger MUST attach an image file to a message
-and MUST add the header `Chat-User-Avatar`
-with the value set to the image name.
+the messenger MUST add the header `Chat-User-Avatar: base64:IMAGEDATA`.
+To bypass limits of headers, it is recommended not to use the outer header
+and to limit the size to 20k.
 
 To remove the profile-image,
 the messenger MUST add the header `Chat-User-Avatar: 0`.
@@ -320,19 +320,14 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     From: sender@domain
     To: rcpt@domain
     Chat-Version: 1.0
-    Chat-User-Avatar: photo.jpg
     Subject: Chat: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 
     --==break==
     Content-Type: text/plain
+    Chat-User-Avatar: base64:AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQY ...
 
     Hello, I've changed my profile image.
-    --==break==
-    Content-Type: image/jpeg
-    Content-Disposition: attachment; filename="photo.jpg"
-
-    AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQYFBA ...
     --==break==--
 
 The image format SHOULD be image/jpeg or image/png.
@@ -342,6 +337,11 @@ in the same message.
 To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
+In older specs, the profile-image was sent as an attachment
+and `Chat-User-Avatar:` specified its name.
+However, it turned out that these attachements are kind of unuexpected to users,
+therefore the profile-image go to the header now.
+
 
 # Locations
 
@@ -401,9 +401,41 @@ it is fine if the location is detected on forwarding etc.
     </kml>
 
 
-# Miscellaneous
+# Stickers
 
-Messengers SHOULD use the header `In-Reply-To` as usual.
+Stickers are send as normal images
+with the additional header `Chat-Content: sticker`.
+
+It is discouraged to send stickers together with user generated text,
+however, stickers can be used as a reply to a message
+and also the footer should be set as usual.
+
+    From: alice@example.org
+    To: bob@example.com
+    Chat-Version: 1.0
+    Chat-Content: sticker
+    Message-ID: Mr.12345uvwxyZ.0005@example.org
+    Subject: Message from Alice
+    Content-Type: multipart/mixed; boundary="==break=="
+
+    --==break==
+    Content-Type: text/plain
+
+    -- 
+    Hi there! I am using this new messenger!
+    --==break==
+    Content-Type: image/png
+    Content-Disposition: attachment; filename="sticker.png"
+
+    R0lGODlhpAGkAfe9AP+zd2eQkZhrI//z9v++PMb///+scrdDT3BtbtrZ2f/LQSsREcdIVf9 ...
+    --==break==--
+
+Typical sticker formats are `image/png`, `image/gif` and `image/webp`.
+Animated stickers are supported
+by just using an image format that supports animation.
+
+
+# Voice messages
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -417,6 +449,11 @@ This allows the receiver to show the time without knowing the file format.
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 
+
+# Miscellaneous
+
+Messengers SHOULD use the header `In-Reply-To` as usual.
+
 Messengers MAY send and receive Message Disposition Notifications
 (MDNs, [RFC 8098](https://tools.ietf.org/html/rfc8098),
 [RFC 3503](https://tools.ietf.org/html/rfc3503))
@@ -437,4 +474,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2020 Delta Chat contributors.
+Copyright © 2017-2021 Delta Chat contributors.

src/accounts.rs

@@ -1,5 +1,8 @@
+//! # Account manager module.
+
 use std::collections::BTreeMap;
 
+use async_std::channel::{self, Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
@@ -18,6 +21,14 @@ pub struct Accounts {
     dir: PathBuf,
     config: Config,
     accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
+    emitter: EventEmitter,
+
+    /// Sender side of the fake event channel.
+    ///
+    /// We never send any events over this channel, but hold it during the account manager lifetime
+    /// to prevent `EventEmitter` from returning `None` as long as account manager is alive, even if
+    /// it holds no accounts which could emit events.
+    fake_sender: Sender<crate::events::Event>,
 }
 
 impl Accounts {
@@ -30,19 +41,13 @@ impl Accounts {
         Accounts::open(dir).await
     }
 
-    /// Creates a new default structure, including a default account.
+    /// Creates a new default structure.
     pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        // create default account
-        let config = Config::new(os_name.clone(), dir).await?;
-        let account_config = config.new_account(dir).await?;
-
-        Context::new(os_name, account_config.dbfile().into(), account_config.id)
-            .await
-            .context("failed to create default account")?;
+        Config::new(os_name.clone(), dir).await?;
 
         Ok(())
     }
@@ -58,10 +63,22 @@ impl Accounts {
         let config = Config::from_file(config_file).await?;
         let accounts = config.load_accounts().await?;
 
+        let emitter = EventEmitter::new();
+
+        // Fake event stream to prevent event emitter from closing.
+        let (fake_sender, fake_receiver) = channel::bounded(1);
+        emitter.sender.send(fake_receiver).await?;
+
+        for account in accounts.values() {
+            emitter.add_account(account).await?;
+        }
+
         Ok(Self {
             dir,
             config,
             accounts: Arc::new(RwLock::new(accounts)),
+            emitter,
+            fake_sender,
         })
     }
 
@@ -71,14 +88,17 @@ impl Accounts {
     }
 
     /// Get the currently selected account.
-    pub async fn get_selected_account(&self) -> Context {
+    pub async fn get_selected_account(&self) -> Option<Context> {
         let id = self.config.get_selected_account().await;
-        self.accounts
-            .read()
-            .await
-            .get(&id)
-            .cloned()
-            .expect("inconsistent state")
+        self.accounts.read().await.get(&id).cloned()
+    }
+
+    /// Returns the currently selected account's id or None if no account is selected.
+    pub async fn get_selected_account_id(&self) -> Option<u32> {
+        match self.config.get_selected_account().await {
+            0 => None,
+            id => Some(id),
+        }
     }
 
     /// Select the given account.
@@ -94,6 +114,7 @@ impl Accounts {
         let account_config = self.config.new_account(&self.dir).await?;
 
         let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.emitter.add_account(&ctx).await?;
         self.accounts.write().await.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -120,6 +141,7 @@ impl Accounts {
     /// Migrate an existing account into this structure.
     pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
+        let walfile = Context::derive_walfile(&dbfile);
 
         ensure!(
             dbfile.exists().await,
@@ -143,6 +165,7 @@ impl Accounts {
 
         let new_dbfile = account_config.dbfile().into();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
+        let new_walfile = Context::derive_walfile(&new_dbfile);
 
         let res = {
             fs::create_dir_all(&account_config.dir)
@@ -154,6 +177,11 @@ impl Accounts {
             fs::rename(&blobdir, &new_blobdir)
                 .await
                 .context("failed to rename blobdir")?;
+            if walfile.exists().await {
+                fs::rename(&walfile, &new_walfile)
+                    .await
+                    .context("failed to rename walfile")?;
+            }
             Ok(())
         };
 
@@ -166,6 +194,7 @@ impl Accounts {
                     account_config.id,
                 )
                 .await?;
+                self.emitter.add_account(&ctx).await?;
                 self.accounts.write().await.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
@@ -190,23 +219,23 @@ impl Accounts {
         self.accounts.read().await.keys().copied().collect()
     }
 
-    /// Import a backup using a new account and selects it.
-    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
-        let old_id = self.config.get_selected_account().await;
-
-        let id = self.add_account().await?;
-        let ctx = self.get_account(id).await.expect("just added");
-
-        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
-            Ok(_) => Ok(id),
-            Err(err) => {
-                // remove temp account
-                self.remove_account(id).await?;
-                // set selection back
-                self.select_account(old_id).await?;
-                Err(err)
+    /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
+    ///
+    /// Returns whether all accounts finished their background work.
+    /// DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
+    ///
+    /// iOS can:
+    /// - call dc_start_io() (in case IO was not running)
+    /// - call dc_maybe_network()
+    /// - while dc_accounts_all_work_done() returns false:
+    ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
+    pub async fn all_work_done(&self) -> bool {
+        for account in self.accounts.read().await.values() {
+            if !account.all_work_done().await {
+                return false;
             }
         }
+        true
     }
 
     pub async fn start_io(&self) {
@@ -230,32 +259,72 @@ impl Accounts {
         }
     }
 
-    /// Unified event emitter.
-    pub async fn get_event_emitter(&self) -> EventEmitter {
-        let emitters: Vec<_> = self
-            .accounts
-            .read()
-            .await
-            .iter()
-            .map(|(_id, a)| a.get_event_emitter())
-            .collect();
+    pub async fn maybe_network_lost(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.maybe_network_lost().await;
+        }
+    }
 
-        EventEmitter(futures::stream::select_all(emitters))
+    /// Returns unified event emitter.
+    pub async fn get_event_emitter(&self) -> EventEmitter {
+        self.emitter.clone()
     }
 }
 
-#[derive(Debug)]
-pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+/// Unified event emitter for multiple accounts.
+#[derive(Debug, Clone)]
+pub struct EventEmitter {
+    /// Aggregate stream of events from all accounts.
+    stream: Arc<RwLock<futures::stream::SelectAll<Receiver<crate::events::Event>>>>,
+
+    /// Sender for the channel where new account emitters will be pushed.
+    sender: Sender<Receiver<crate::events::Event>>,
+
+    /// Receiver for the channel where new account emitters will be pushed.
+    receiver: Receiver<Receiver<crate::events::Event>>,
+}
 
 impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
-    pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv())
+    pub fn new() -> Self {
+        let (sender, receiver) = channel::unbounded();
+        Self {
+            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
+            sender,
+            receiver,
+        }
     }
 
-    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
-    pub async fn recv(&mut self) -> Option<Event> {
-        self.0.next().await
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
+        async_std::task::block_on(self.recv()).unwrap_or_default()
+    }
+
+    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
+    pub async fn recv(&mut self) -> Result<Option<Event>> {
+        let mut stream = self.stream.write().await;
+        loop {
+            match futures::future::select(self.receiver.recv(), stream.next()).await {
+                futures::future::Either::Left((emitter, _)) => {
+                    stream.push(emitter?);
+                }
+                futures::future::Either::Right((ev, _)) => return Ok(ev),
+            }
+        }
+    }
+
+    /// Add event emitter of a new account to the aggregate event emitter.
+    pub async fn add_account(&self, context: &Context) -> Result<()> {
+        self.sender
+            .send(context.get_event_emitter().into_inner())
+            .await?;
+        Ok(())
+    }
+}
+
+impl Default for EventEmitter {
+    fn default() -> Self {
+        Self::new()
     }
 }
 
@@ -266,13 +335,14 @@ impl async_std::stream::Stream for EventEmitter {
         mut self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self.0).poll_next(cx)
+        std::pin::Pin::new(&mut self).poll_next(cx)
     }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
+/// Account manager configuration file.
 #[derive(Debug, Clone)]
 pub struct Config {
     file: PathBuf,
@@ -347,7 +417,7 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+    async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
             let inner = &mut self.inner.write().await;
             let id = inner.next_id;
@@ -387,7 +457,7 @@ impl Config {
         self.sync().await
     }
 
-    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+    async fn get_account(&self, id: u32) -> Option<AccountConfig> {
         self.inner
             .read()
             .await
@@ -418,8 +488,9 @@ impl Config {
     }
 }
 
+/// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub struct AccountConfig {
+struct AccountConfig {
     /// Unique id.
     pub id: u32,
     /// Root directory for all data for this account.
@@ -444,6 +515,8 @@ mod tests {
         let p: PathBuf = dir.path().join("accounts1").into();
 
         let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        accounts1.add_account().await.unwrap();
+
         let accounts2 = Accounts::open(p).await.unwrap();
 
         assert_eq!(accounts1.accounts.read().await.len(), 1);
@@ -466,7 +539,11 @@ mod tests {
         let p: PathBuf = dir.path().join("accounts").into();
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 0);
+        assert_eq!(accounts.config.get_selected_account().await, 0);
 
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, 1);
         assert_eq!(accounts.accounts.read().await.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
@@ -483,14 +560,35 @@ mod tests {
         assert_eq!(accounts.accounts.read().await.len(), 1);
     }
 
+    #[async_std::test]
+    async fn test_accounts_remove_last() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+        assert!(accounts.get_selected_account().await.is_none());
+        assert_eq!(accounts.config.get_selected_account().await, 0);
+
+        let id = accounts.add_account().await?;
+        assert!(accounts.get_selected_account().await.is_some());
+        assert_eq!(id, 1);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, id);
+
+        accounts.remove_account(id).await?;
+        assert!(accounts.get_selected_account().await.is_none());
+
+        Ok(())
+    }
+
     #[async_std::test]
     async fn test_migrate_account() {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
+        assert_eq!(accounts.accounts.read().await.len(), 0);
+        assert_eq!(accounts.config.get_selected_account().await, 0);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
         let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
@@ -506,10 +604,10 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 2);
-        assert_eq!(accounts.config.get_selected_account().await, 2);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
 
-        let ctx = accounts.get_selected_account().await;
+        let ctx = accounts.get_selected_account().await.unwrap();
         assert_eq!(
             "me@mail.com",
             ctx.get_config(crate::config::Config::Addr)
@@ -527,7 +625,7 @@ mod tests {
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        for expected_id in 2..10 {
+        for expected_id in 1..10 {
             let id = accounts.add_account().await.unwrap();
             assert_eq!(id, expected_id);
         }
@@ -537,4 +635,112 @@ mod tests {
             assert_eq!(ids.get(i), Some(&expected_id));
         }
     }
+
+    #[async_std::test]
+    async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
+        let dir = tempfile::tempdir()?;
+        let p: PathBuf = dir.path().join("accounts").into();
+        let dummy_accounts = 10;
+
+        let (id0, id1, id2) = {
+            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            accounts.add_account().await?;
+            let ids = accounts.get_all().await;
+            assert_eq!(ids.len(), 1);
+
+            let id0 = *ids.get(0).unwrap();
+            let ctx = accounts.get_account(id0).await.unwrap();
+            ctx.set_config(crate::config::Config::Addr, Some("one@example.org"))
+                .await?;
+
+            let id1 = accounts.add_account().await?;
+            let ctx = accounts.get_account(id1).await.unwrap();
+            ctx.set_config(crate::config::Config::Addr, Some("two@example.org"))
+                .await?;
+
+            // add and remove some accounts and force a gap (ids must not be reused)
+            for _ in 0..dummy_accounts {
+                let to_delete = accounts.add_account().await?;
+                accounts.remove_account(to_delete).await?;
+            }
+
+            let id2 = accounts.add_account().await?;
+            let ctx = accounts.get_account(id2).await.unwrap();
+            ctx.set_config(crate::config::Config::Addr, Some("three@example.org"))
+                .await?;
+
+            accounts.select_account(id1).await?;
+
+            (id0, id1, id2)
+        };
+        assert!(id0 > 0);
+        assert!(id1 > id0);
+        assert!(id2 > id1 + dummy_accounts);
+
+        let (id0_reopened, id1_reopened, id2_reopened) = {
+            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+            let ctx = accounts.get_selected_account().await.unwrap();
+            assert_eq!(
+                ctx.get_config(crate::config::Config::Addr).await?,
+                Some("two@example.org".to_string())
+            );
+
+            let ids = accounts.get_all().await;
+            assert_eq!(ids.len(), 3);
+
+            let id0 = *ids.get(0).unwrap();
+            let ctx = accounts.get_account(id0).await.unwrap();
+            assert_eq!(
+                ctx.get_config(crate::config::Config::Addr).await?,
+                Some("one@example.org".to_string())
+            );
+
+            let id1 = *ids.get(1).unwrap();
+            let t = accounts.get_account(id1).await.unwrap();
+            assert_eq!(
+                t.get_config(crate::config::Config::Addr).await?,
+                Some("two@example.org".to_string())
+            );
+
+            let id2 = *ids.get(2).unwrap();
+            let ctx = accounts.get_account(id2).await.unwrap();
+            assert_eq!(
+                ctx.get_config(crate::config::Config::Addr).await?,
+                Some("three@example.org".to_string())
+            );
+
+            (id0, id1, id2)
+        };
+        assert_eq!(id0, id0_reopened);
+        assert_eq!(id1, id1_reopened);
+        assert_eq!(id2, id2_reopened);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_no_accounts_event_emitter() -> Result<()> {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
+
+        // Make sure there are no accounts.
+        assert_eq!(accounts.accounts.read().await.len(), 0);
+
+        // Create event emitter.
+        let mut event_emitter = accounts.get_event_emitter().await;
+
+        // Test that event emitter does not return `None` immediately.
+        let duration = std::time::Duration::from_millis(1);
+        assert!(async_std::future::timeout(duration, event_emitter.recv())
+            .await
+            .is_err());
+
+        // When account manager is dropped, event emitter is exhausted.
+        drop(accounts);
+        assert_eq!(event_emitter.recv().await?, None);
+
+        Ok(())
+    }
 }

src/aheader.rs

@@ -1,4 +1,4 @@
-//! # Autocrypt header module
+//! # Autocrypt header module.
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 

src/blob.rs

@@ -1,5 +1,6 @@
-//! # Blob directory management
+//! # Blob directory management.
 
+use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 
@@ -7,8 +8,12 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::format_err;
+use anyhow::Context as _;
 use anyhow::Error;
+use image::DynamicImage;
 use image::GenericImageView;
+use image::ImageFormat;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
@@ -53,11 +58,11 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: impl AsRef<str>,
+        suggested_name: &str,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
         let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
@@ -69,7 +74,7 @@ impl<'a> BlobObject<'a> {
 
         // workaround a bug in async-std
         // (the executor does not handle blocking operation in Drop correctly,
-        // see https://github.com/async-rs/async-std/issues/900 )
+        // see <https://github.com/async-rs/async-std/issues/900>)
         let _ = file.flush().await;
 
         let blob = BlobObject {
@@ -132,18 +137,17 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: impl AsRef<Path>,
+        src: &Path,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file =
-            fs::File::open(src.as_ref())
-                .await
-                .map_err(|err| BlobError::CopyFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: String::from(""),
-                    src: src.as_ref().to_path_buf(),
-                    cause: err,
-                })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let mut src_file = fs::File::open(src)
+            .await
+            .map_err(|err| BlobError::CopyFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: String::from(""),
+                src: src.to_path_buf(),
+                cause: err,
+            })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
@@ -156,7 +160,7 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.as_ref().to_path_buf(),
+                src: src.to_path_buf(),
                 cause: err,
             });
         }
@@ -190,16 +194,13 @@ impl<'a> BlobObject<'a> {
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &Context,
-        src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject<'_>, BlobError> {
-        if src.as_ref().starts_with(context.get_blobdir()) {
+        context: &'a Context,
+        src: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        if src.starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.as_ref().starts_with("$BLOBDIR/") {
-            BlobObject::from_name(
-                context,
-                src.as_ref().to_str().unwrap_or_default().to_string(),
-            )
+        } else if src.starts_with("$BLOBDIR/") {
+            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -220,23 +221,22 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &Context,
-        path: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject, BlobError> {
-        let rel_path = path
-            .as_ref()
-            .strip_prefix(context.get_blobdir())
-            .map_err(|_| BlobError::WrongBlobdir {
-                blobdir: context.get_blobdir().to_path_buf(),
-                src: path.as_ref().to_path_buf(),
-            })?;
-        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+        context: &'a Context,
+        path: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let rel_path =
+            path.strip_prefix(context.get_blobdir())
+                .map_err(|_| BlobError::WrongBlobdir {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    src: path.to_path_buf(),
+                })?;
+        if !BlobObject::is_acceptible_blob_name(rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.as_ref().to_path_buf(),
+                blobname: path.to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.as_ref().to_path_buf(),
+            blobname: path.to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -380,7 +380,7 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
 
         let img_wh =
@@ -391,7 +391,15 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_AVATAR_SIZE,
             };
 
-        self.recode_to_size(context, blob_abs, img_wh).await
+        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
+        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
+        if let Some(new_name) = self
+            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
+            .await?
+        {
+            self.name = new_name;
+        }
+        Ok(())
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -410,30 +418,69 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_IMAGE_SIZE,
             };
 
-        self.recode_to_size(context, blob_abs, img_wh).await
+        if self
+            .recode_to_size(context, blob_abs, img_wh, None)
+            .await?
+            .is_some()
+        {
+            return Err(format_err!(
+                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
+            )
+            .into());
+        }
+        Ok(())
     }
 
     async fn recode_to_size(
         &self,
         context: &Context,
-        blob_abs: PathBuf,
-        img_wh: u32,
-    ) -> Result<(), BlobError> {
+        mut blob_abs: PathBuf,
+        mut img_wh: u32,
+        max_bytes: Option<usize>,
+    ) -> Result<Option<String>, BlobError> {
         let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
         let orientation = self.get_exif_orientation(context);
+        let mut encoded = Vec::new();
+        let mut changed_name = None;
 
-        let do_scale = img.width() > img_wh || img.height() > img_wh;
+        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
+            encoded.clear();
+            img.write_to(encoded, image::ImageFormat::Jpeg)?;
+            Ok(())
+        }
+        fn encoded_img_exceeds_bytes(
+            context: &Context,
+            img: &DynamicImage,
+            max_bytes: Option<usize>,
+            encoded: &mut Vec<u8>,
+        ) -> anyhow::Result<bool> {
+            if let Some(max_bytes) = max_bytes {
+                encode_img(img, encoded)?;
+                if encoded.len() > max_bytes {
+                    info!(
+                        context,
+                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
+                        encoded.len(),
+                        img.width(),
+                        img.height(),
+                        max_bytes,
+                    );
+                    return Ok(true);
+                }
+            }
+            Ok(false)
+        }
+        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
+
+        let do_scale =
+            exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
         let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
         if do_scale || do_rotate {
-            if do_scale {
-                img = img.thumbnail(img_wh, img_wh);
-            }
-
             if do_rotate {
                 img = match orientation {
                     Ok(90) => img.rotate90(),
@@ -443,14 +490,64 @@ impl<'a> BlobObject<'a> {
                 }
             }
 
-            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                cause: err.into(),
-            })?;
+            if do_scale {
+                if !exceeds_width {
+                    // The image is already smaller than img_wh, but exceeds max_bytes
+                    // We can directly start with trying to scale down to 2/3 of its current width
+                    img_wh = max(img.width(), img.height()) * 2 / 3
+                }
+
+                loop {
+                    let new_img = img.thumbnail(img_wh, img_wh);
+
+                    if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
+                        if img_wh < 20 {
+                            return Err(format_err!(
+                                "Failed to scale image to below {}B",
+                                max_bytes.unwrap_or_default()
+                            )
+                            .into());
+                        }
+
+                        img_wh = img_wh * 2 / 3;
+                    } else {
+                        if encoded.is_empty() {
+                            encode_img(&new_img, &mut encoded)?;
+                        }
+
+                        info!(
+                            context,
+                            "Final scaled-down image size: {}B ({}px)",
+                            encoded.len(),
+                            img_wh
+                        );
+                        break;
+                    }
+                }
+            }
+
+            // The file format is JPEG now, we may have to change the file extension
+            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
+                blob_abs = blob_abs.with_extension("jpg");
+                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
+                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
+                changed_name = Some(format!("$BLOBDIR/{}", file_name));
+            }
+
+            if encoded.is_empty() {
+                encode_img(&img, &mut encoded)?;
+            }
+
+            fs::write(&blob_abs, &encoded)
+                .await
+                .map_err(|err| BlobError::WriteFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                    cause: err.into(),
+                })?;
         }
 
-        Ok(())
+        Ok(changed_name)
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -514,17 +611,18 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
-    #[error("Sql: {0}")]
-    Sql(#[from] crate::sql::Error),
     #[error("{0}")]
     Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
+    use fs::File;
+
     use super::*;
 
-    use crate::test_utils::TestContext;
+    use crate::{message::Message, test_utils::TestContext};
+    use image::Pixel;
 
     #[async_std::test]
     async fn test_create() {
@@ -626,13 +724,15 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
+        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
+            .await
+            .is_err());
         let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
@@ -643,7 +743,9 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -660,7 +762,9 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -715,4 +819,249 @@ mod tests {
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(&avatar_blob).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+
+        async fn file_size(path_buf: &PathBuf) -> u64 {
+            let file = File::open(path_buf).await.unwrap();
+            file.metadata().await.unwrap().len()
+        }
+
+        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
+
+        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
+            .await
+            .unwrap();
+        assert!(file_size(&avatar_blob).await <= 3000);
+        assert!(file_size(&avatar_blob).await > 2000);
+        let img = image::open(&avatar_blob).unwrap();
+        assert!(img.width() > 130);
+        assert_eq!(img.width(), img.height());
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
+        assert_eq!(
+            avatar_cfg,
+            avatar_src.with_extension("jpg").to_str().unwrap()
+        );
+
+        let img = image::open(avatar_cfg).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
+
+    #[async_std::test]
+    async fn test_recode_image() {
+        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
+        send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
+            .await
+            .unwrap();
+        send_image_check_mediaquality(
+            Some("1"),
+            bytes,
+            1000,
+            1000,
+            0,
+            WORSE_IMAGE_SIZE,
+            WORSE_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+
+        // The "-rotated" files are rotated by 270 degrees using the Exif metadata
+        let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
+        let img_rotated = send_image_check_mediaquality(
+            Some("0"),
+            bytes,
+            2000,
+            1800,
+            270,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+        assert_correct_rotation(&img_rotated);
+
+        let mut bytes = vec![];
+        img_rotated
+            .write_to(&mut bytes, image::ImageFormat::Jpeg)
+            .unwrap();
+        let img_rotated = send_image_check_mediaquality(
+            Some("0"),
+            &bytes,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+            0,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+        assert_correct_rotation(&img_rotated);
+
+        let img_rotated = send_image_check_mediaquality(
+            Some("1"),
+            &bytes,
+            BALANCED_IMAGE_SIZE * 1800 / 2000,
+            BALANCED_IMAGE_SIZE,
+            0,
+            WORSE_IMAGE_SIZE * 1800 / 2000,
+            WORSE_IMAGE_SIZE,
+        )
+        .await
+        .unwrap();
+        assert_correct_rotation(&img_rotated);
+
+        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
+        let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
+            .await
+            .unwrap();
+        assert_correct_rotation(&img_rotated);
+
+        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
+        let img_rotated = send_image_check_mediaquality(Some("1"), bytes, 200, 180, 270, 180, 200)
+            .await
+            .unwrap();
+        assert_correct_rotation(&img_rotated);
+    }
+
+    fn assert_correct_rotation(img: &DynamicImage) {
+        // The test images are black in the bottom left corner after correctly applying
+        // the EXIF orientation
+
+        let [luma] = img.get_pixel(10, 10).to_luma().0;
+        assert_eq!(luma, 255);
+        let [luma] = img.get_pixel(img.width() - 10, 10).to_luma().0;
+        assert_eq!(luma, 255);
+        let [luma] = img
+            .get_pixel(img.width() - 10, img.height() - 10)
+            .to_luma()
+            .0;
+        assert_eq!(luma, 255);
+        let [luma] = img.get_pixel(10, img.height() - 10).to_luma().0;
+        assert_eq!(luma, 0);
+    }
+
+    async fn send_image_check_mediaquality(
+        media_quality_config: Option<&str>,
+        bytes: &[u8],
+        original_width: u32,
+        original_height: u32,
+        orientation: i32,
+        compressed_width: u32,
+        compressed_height: u32,
+    ) -> anyhow::Result<DynamicImage> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        alice
+            .set_config(Config::MediaQuality, media_quality_config)
+            .await?;
+        let file = alice.get_blobdir().join("file.jpg");
+
+        File::create(&file).await?.write_all(bytes).await?;
+        let img = image::open(&file)?;
+        assert_eq!(img.width(), original_width);
+        assert_eq!(img.height(), original_height);
+
+        let blob = BlobObject::new_from_path(&alice, &file).await?;
+        assert_eq!(blob.get_exif_orientation(&alice).unwrap_or(0), orientation);
+
+        let mut msg = Message::new(Viewtype::Image);
+        msg.set_file(file.to_str().unwrap(), None);
+        let chat = alice.create_chat(&bob).await;
+        let sent = alice.send_msg(chat.id, &mut msg).await;
+        let alice_msg = alice.get_last_msg().await;
+        assert_eq!(alice_msg.get_width() as u32, compressed_width);
+        assert_eq!(alice_msg.get_height() as u32, compressed_height);
+        let img = image::open(alice_msg.get_file(&alice).unwrap())?;
+        assert_eq!(img.width() as u32, compressed_width);
+        assert_eq!(img.height() as u32, compressed_height);
+
+        bob.recv_msg(&sent).await;
+        let bob_msg = bob.get_last_msg().await;
+        assert_eq!(bob_msg.get_width() as u32, compressed_width);
+        assert_eq!(bob_msg.get_height() as u32, compressed_height);
+        let file = bob_msg.get_file(&bob).unwrap();
+
+        let blob = BlobObject::new_from_path(&bob, &file).await?;
+        assert_eq!(blob.get_exif_orientation(&bob).unwrap_or(0), 0);
+
+        let img = image::open(file)?;
+        assert_eq!(img.width() as u32, compressed_width);
+        assert_eq!(img.height() as u32, compressed_height);
+        Ok(img)
+    }
 }

src/chatlist.rs

@@ -1,15 +1,12 @@
-//! # Chat list module
+//! # Chat list module.
 
 use anyhow::{bail, ensure, Result};
-use async_std::prelude::*;
-use sqlx::Row;
 
-use crate::chat;
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
-    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
-    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
+    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
+    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
 use crate::contact::Contact;
 use crate::context::Context;
@@ -37,15 +34,12 @@ use crate::stock_str;
 /// and for each messages that is scrolled into view, dc_get_msg() is called then.
 ///
 /// Why no listflags?
-/// Without listflags, dc_get_chatlist() adds the deaddrop and the archive "link" automatically as needed.
-/// The UI can just render these items differently then. Although the deaddrop link is currently always the
-/// first entry and only present on new messages, there is the rough idea that it can be optionally always
-/// present and sorted into the list by date. Rendering the deaddrop in the described way
-/// would not add extra work in the UI then.
+/// Without listflags, dc_get_chatlist() adds the archive "link" automatically as needed.
+/// The UI can just render these items differently then.
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(ChatId, MsgId)>,
+    ids: Vec<(ChatId, Option<MsgId>)>,
 }
 
 impl Chatlist {
@@ -61,12 +55,6 @@ impl Chatlist {
     ///
     /// By default, the function adds some special entries to the list.
     /// These special entries can be identified by the ID returned by chatlist.get_chat_id():
-    /// - DC_CHAT_ID_DEADDROP (1) - this special chat is present if there are
-    ///   messages from addresses that have no relationship to the configured account.
-    ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
-    ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Start chat", "Block" and "Not now";
-    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -82,9 +70,9 @@ impl Chatlist {
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat,
+    ///   and hides the device-chat and contact requests
     ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
-    /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
+    /// - if the flag DC_GCL_NO_SPECIALS is set, archive link is not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
@@ -112,20 +100,23 @@ impl Chatlist {
 
         let mut add_archived_link_item = false;
 
-        let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .await
-                .unwrap_or_default()
-                .0
-        } else {
-            ChatId::new(0)
+        let process_row = |row: &rusqlite::Row| {
+            let chat_id: ChatId = row.get(0)?;
+            let msg_id: Option<MsgId> = row.get(1)?;
+            Ok((chat_id, msg_id))
         };
 
-        let process_row = |row: sqlx::Result<sqlx::sqlite::SqliteRow>| {
-            let row = row?;
-            let chat_id: ChatId = row.try_get(0)?;
-            let msg_id: MsgId = row.try_get(1).unwrap_or_default();
-            Ok((chat_id, msg_id))
+        let process_rows = |rows: rusqlite::MappedRows<_>| {
+            rows.collect::<std::result::Result<Vec<_>, _>>()
+                .map_err(Into::into)
+        };
+
+        let skip_id = if flag_for_forwarding {
+            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+                .await?
+                .unwrap_or_default()
+        } else {
+            ChatId::new(0)
         };
 
         // select with left join and minimum:
@@ -136,17 +127,12 @@ impl Chatlist {
         //   timestamp
         // - the list starts with the newest chats
         //
-        // nb: the query currently shows messages from blocked
-        // contacts in groups.  however, for normal-groups, this is
-        // okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and
-        // tg do the same) for the deaddrop, however, they should
-        // really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
-        let mut ids: Vec<_> = if let Some(query_contact_id) = query_contact_id {
+        // The query shows messages from blocked contacts in
+        // groups. Otherwise it would be hard to follow conversations.
+        let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.fetch(
-                sqlx::query("SELECT c.id, m.id
+            context.sql.query_map(
+                "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -157,12 +143,14 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked=0
+                   AND c.blocked!=1
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;"
-                ).bind(MessageState::OutDraft).bind(query_contact_id).bind(ChatVisibility::Pinned)
-            ).await?.map(process_row).collect::<sqlx::Result<_>>().await?
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                process_row,
+                process_rows,
+            ).await?
         } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
@@ -170,9 +158,8 @@ impl Chatlist {
             // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.id, m.id
+                .query_map(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -183,17 +170,15 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked=0
+                   AND c.blocked!=1
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    )
-                    .bind(MessageState::OutDraft),
+                    paramsv![MessageState::OutDraft],
+                    process_row,
+                    process_rows,
                 )
                 .await?
-                .map(process_row)
-                .collect::<sqlx::Result<_>>()
-                .await?
         } else if let Some(query) = query {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
@@ -207,9 +192,8 @@ impl Chatlist {
             let str_like_cmd = format!("%{}%", query);
             context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.id, m.id
+                .query_map(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -220,31 +204,25 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND c.blocked=0
+                   AND c.blocked!=1
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    )
-                    .bind(MessageState::OutDraft)
-                    .bind(skip_id)
-                    .bind(str_like_cmd),
+                    paramsv![MessageState::OutDraft, skip_id, str_like_cmd],
+                    process_row,
+                    process_rows,
                 )
                 .await?
-                .map(process_row)
-                .collect::<sqlx::Result<_>>()
-                .await?
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .await
+                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                    .await?
                     .unwrap_or_default()
-                    .0
             } else {
                 ChatId::new(0)
             };
-
-            let mut ids: Vec<_> = context.sql.fetch(sqlx::query(
+            let ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -256,26 +234,15 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND c.blocked=0
-                   AND NOT c.archived=?3
+                   AND (c.blocked=0 OR (c.blocked=2 AND NOT ?3))
+                   AND NOT c.archived=?4
                  GROUP BY c.id
-                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;"
-            )
-              .bind(MessageState::OutDraft)
-              .bind(skip_id)
-              .bind(ChatVisibility::Archived)
-              .bind(sort_id_up)
-              .bind(ChatVisibility::Pinned)
-            ).await?.map(process_row).collect::<sqlx::Result<_>>().await?;
-
+                 ORDER BY c.id=?5 DESC, c.archived=?6 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, skip_id, flag_for_forwarding, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                process_row,
+                process_rows,
+            ).await?;
             if !flag_no_specials {
-                if let Some(last_deaddrop_fresh_msg_id) =
-                    get_last_deaddrop_fresh_msg(context).await?
-                {
-                    if !flag_for_forwarding {
-                        ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
-                    }
-                }
                 add_archived_link_item = true;
             }
             ids
@@ -283,9 +250,9 @@ impl Chatlist {
 
         if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, None));
         }
 
         Ok(Chatlist { ids })
@@ -314,7 +281,7 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
+    pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
         match self.ids.get(index) {
             Some((_chat_id, msg_id)) => Ok(*msg_id),
             None => bail!("Chatlist index out of range"),
@@ -335,18 +302,19 @@ impl Chatlist {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
+    pub async fn get_summary(
+        &self,
+        context: &Context,
+        index: usize,
+        chat: Option<&Chat>,
+    ) -> Result<Lot> {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
-            None => {
-                let mut ret = Lot::new();
-                ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return Lot::new();
-            }
+            None => bail!("Chatlist index out of range"),
         };
 
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
@@ -355,50 +323,50 @@ impl Chatlist {
     pub async fn get_summary2(
         context: &Context,
         chat_id: ChatId,
-        lastmsg_id: MsgId,
+        lastmsg_id: Option<MsgId>,
         chat: Option<&Chat>,
-    ) -> Lot {
+    ) -> Result<Lot> {
         let mut ret = Lot::new();
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
+        } else {
+            let chat = Chat::load_from_db(context, chat_id).await?;
             chat_loaded = chat;
             &chat_loaded
-        } else {
-            return ret;
         };
 
-        let (lastmsg, lastcontact) =
-            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-                if lastmsg.from_id == DC_CONTACT_ID_SELF {
-                    (Some(lastmsg), None)
-                } else {
-                    match chat.typ {
-                        Chattype::Group | Chattype::Mailinglist => {
-                            let lastcontact =
-                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
-                            (Some(lastmsg), lastcontact)
-                        }
-                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
-                    }
-                }
+        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
+            let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
+            if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                (Some(lastmsg), None)
             } else {
-                (None, None)
-            };
+                match chat.typ {
+                    Chattype::Group | Chattype::Mailinglist => {
+                        let lastcontact =
+                            Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                        (Some(lastmsg), lastcontact)
+                    }
+                    Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
+                }
+            }
+        } else {
+            (None, None)
+        };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
-        } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
+        } else if let Some(mut lastmsg) =
+            lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED)
         {
-            ret.text2 = Some(stock_str::no_messages(context).await);
-        } else {
-            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
+            ret.fill(&mut lastmsg, chat, lastcontact.as_ref(), context)
                 .await;
+        } else {
+            ret.text2 = Some(stock_str::no_messages(context).await);
         }
 
-        ret
+        Ok(ret)
     }
 
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
@@ -410,32 +378,14 @@ impl Chatlist {
 pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
     let count = context
         .sql
-        .count(sqlx::query(
-            "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
-        ))
+        .count(
+            "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
+            paramsv![Blocked::Manually, ChatVisibility::Archived],
+        )
         .await?;
     Ok(count)
 }
 
-async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>> {
-    // We have an index over the state-column, this should be
-    // sufficient as there are typically only few fresh messages.
-    let id = context
-        .sql
-        .query_get_value(sqlx::query(concat!(
-            "SELECT m.id",
-            " FROM msgs m",
-            " LEFT JOIN chats c",
-            "        ON c.id=m.chat_id",
-            " WHERE m.state=10",
-            "   AND m.hidden=0",
-            "   AND c.blocked=2",
-            " ORDER BY m.timestamp DESC, m.id DESC;"
-        )))
-        .await?;
-    Ok(id)
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -443,8 +393,6 @@ mod tests {
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
     use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message;
-    use crate::message::ContactRequestDecision;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
@@ -554,7 +502,7 @@ mod tests {
     async fn test_search_single_chat() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
-        // receive a one-to-one-message, accept contact request
+        // receive a one-to-one-message
         dc_receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
@@ -572,15 +520,13 @@ mod tests {
         .await?;
 
         let chats = Chatlist::try_load(&t, 0, Some("Bob Authname"), None).await?;
-        assert_eq!(chats.len(), 0);
+        // Contact request should be searchable
+        assert_eq!(chats.len(), 1);
 
         let msg = t.get_last_msg().await;
-        assert_eq!(msg.get_chat_id(), DC_CHAT_ID_DEADDROP);
+        let chat_id = msg.get_chat_id();
+        chat_id.accept(&t).await.unwrap();
 
-        let chat_id =
-            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
-                .await
-                .unwrap();
         let contacts = get_chat_contacts(&t, chat_id).await?;
         let contact_id = *contacts.first().unwrap();
         let chat = Chat::load_from_db(&t, chat_id).await?;
@@ -618,7 +564,7 @@ mod tests {
     async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
-        // receive a one-to-one-message without authname set, accept contact request
+        // receive a one-to-one-message without authname set
         dc_receive_imf(
             &t,
             b"From: bob@example.org\n\
@@ -636,10 +582,8 @@ mod tests {
         .await?;
 
         let msg = t.get_last_msg().await;
-        let chat_id =
-            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
-                .await
-                .unwrap();
+        let chat_id = msg.get_chat_id();
+        chat_id.accept(&t).await.unwrap();
         let contacts = get_chat_contacts(&t, chat_id).await?;
         let contact_id = *contacts.first().unwrap();
         let chat = Chat::load_from_db(&t, chat_id).await?;
@@ -692,7 +636,7 @@ mod tests {
         chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t, 0, None).await;
+        let summary = chats.get_summary(&t, 0, None).await.unwrap();
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -1,4 +1,4 @@
-//! Implementation of Consistent Color Generation
+//! Implementation of Consistent Color Generation.
 //!
 //! Consistent Color Generation is defined in XEP-0392.
 //!
@@ -8,8 +8,8 @@ use hsluv::hsluv_to_rgb;
 use sha1::{Digest, Sha1};
 
 /// Converts an identifier to Hue angle.
-fn str_to_angle(s: impl AsRef<str>) -> f64 {
-    let bytes = s.as_ref().as_bytes();
+fn str_to_angle(s: &str) -> f64 {
+    let bytes = s.as_bytes();
     let result = Sha1::digest(bytes);
     let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
         + 256 * result.get(1).map_or(0, |&x| u16::from(x));
@@ -31,7 +31,7 @@ fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
 ///
 /// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
 /// half (50.0) to make colors suitable both for light and dark theme.
-pub(crate) fn str_to_color(s: impl AsRef<str>) -> u32 {
+pub(crate) fn str_to_color(s: &str) -> u32 {
     rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
 }
 
@@ -42,7 +42,7 @@ mod tests {
     #[test]
     fn test_str_to_angle() {
         // Test against test vectors from
-        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
+        // <https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd>
         assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
         assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
         assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);

src/config.rs

@@ -1,4 +1,4 @@
-//! # Key-value configuration management
+//! # Key-value configuration management.
 
 use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
@@ -18,7 +18,18 @@ use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
-    Debug, Clone, Copy, PartialEq, Eq, Display, EnumString, AsRefStr, EnumIter, EnumProperty,
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    Display,
+    EnumString,
+    AsRefStr,
+    EnumIter,
+    EnumProperty,
+    PartialOrd,
+    Ord,
 )]
 #[strum(serialize_all = "snake_case")]
 pub enum Config {
@@ -37,6 +48,12 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
+    Socks5Enabled,
+    Socks5Host,
+    Socks5Port,
+    Socks5User,
+    Socks5Password,
+
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -139,6 +156,11 @@ pub enum Config {
     #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
+    /// If a warning about exceeding quota was shown recently,
+    /// this is the percentage of quota at the time the warning was given.
+    /// Unset, when quota falls below minimal warning threshold again.
+    QuotaExceeding,
+
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
 
@@ -242,14 +264,14 @@ impl Context {
         match key {
             Config::Selfavatar => {
                 self.sql
-                    .execute(sqlx::query("UPDATE contacts SET selfavatar_sent=0;"))
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![])
                     .await?;
                 self.sql
                     .set_raw_config_bool("attach_selfavatar", true)
                     .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(self, value).await?;
+                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql.set_raw_config(key, Some(blob.as_name())).await?;
                         Ok(())
@@ -305,7 +327,7 @@ impl Context {
         }
     }
 
-    pub async fn set_config_bool(&self, key: Config, value: bool) -> crate::sql::Result<()> {
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
         self.set_config(key, if value { Some("1") } else { None })
             .await?;
         Ok(())
@@ -331,12 +353,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::BALANCED_AVATAR_SIZE;
     use crate::test_utils::TestContext;
-    use image::GenericImageView;
     use num_traits::FromPrimitive;
-    use std::fs::File;
-    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -350,82 +368,6 @@ mod tests {
         );
     }
 
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;

src/configure.rs

@@ -1,4 +1,4 @@
-//! Email accounts autoconfiguration process module
+//! Email accounts autoconfiguration process module.
 
 mod auto_mozilla;
 mod auto_outlook;
@@ -14,6 +14,7 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
+use crate::login_param::Socks5Config;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
@@ -170,12 +171,17 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
+    let socks5_config = param.socks5_config.clone();
+    let socks5_enabled = socks5_config.is_some();
+
     let ctx2 = ctx.clone();
     let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
-    if oauth2 {
+    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
+    // socks5 requests, this can work with socks5 too
+    if oauth2 && !socks5_enabled {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -217,7 +223,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+        if let Some(provider) = provider::get_provider_info(&param_domain, socks5_enabled).await {
             param.provider = Some(provider);
             match provider.status {
                 provider::Status::Ok | provider::Status::Preparation => {
@@ -256,9 +262,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                 }
             }
         } else {
+            // Try receiving autoconfig
             info!(ctx, "no offline autoconfig found");
-            param_autoconfig =
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
+            param_autoconfig = if socks5_enabled {
+                // Currently we can't do http requests through socks5, to not leak
+                // the ip, just don't do online autoconfig
+                info!(ctx, "socks5 enabled, skipping autoconfig");
+                None
+            } else {
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
+            }
         }
     } else {
         param_autoconfig = None;
@@ -320,6 +333,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             match try_smtp_one_param(
                 &context_smtp,
                 &smtp_param,
+                &socks5_config,
                 &smtp_addr,
                 oauth2,
                 provider_strict_tls,
@@ -345,10 +359,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 600);
 
     // Configure IMAP
-    let (_s, r) = async_std::channel::bounded(1);
-    let mut imap = Imap::new(r);
 
-    let mut imap_configured = false;
+    let mut imap: Option<Imap> = None;
     let imap_servers: Vec<&ServerParams> = servers
         .iter()
         .filter(|params| params.protocol == Protocol::Imap)
@@ -364,15 +376,15 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         match try_imap_one_param(
             ctx,
             &param.imap,
+            &param.socks5_config,
             &param.addr,
             oauth2,
             provider_strict_tls,
-            &mut imap,
         )
         .await
         {
-            Ok(_) => {
-                imap_configured = true;
+            Ok(configured_imap) => {
+                imap = Some(configured_imap);
                 break;
             }
             Err(e) => errors.push(e),
@@ -382,9 +394,10 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
         );
     }
-    if !imap_configured {
-        bail!(nicer_configuration_error(ctx, errors).await);
-    }
+    let mut imap = match imap {
+        Some(imap) => imap,
+        None => bail!(nicer_configuration_error(ctx, errors).await),
+    };
 
     progress!(ctx, 850);
 
@@ -449,7 +462,7 @@ async fn get_autoconfig(
 ) -> Option<Vec<ServerParams>> {
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        format!(
+        &format!(
             "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
             param_domain, param_addr_urlencoded
         ),
@@ -463,8 +476,8 @@ async fn get_autoconfig(
 
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-        format!(
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see <https://releases.mozilla.org/pub/thunderbird/>,  which makes some sense
+        &format!(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
         ),
@@ -503,7 +516,7 @@ async fn get_autoconfig(
     // always SSL for Thunderbird's database
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
         param,
     )
     .await
@@ -517,48 +530,88 @@ async fn get_autoconfig(
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
-    imap: &mut Imap,
-) -> Result<(), ConfigurationError> {
+) -> Result<Imap, ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
         param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = imap
-        .connect(context, param, addr, oauth2, provider_strict_tls)
-        .await
+    let (_s, r) = async_std::channel::bounded(1);
+
+    let mut imap = match Imap::new(
+        param,
+        socks5_config.clone(),
+        addr,
+        oauth2,
+        provider_strict_tls,
+        r,
+    )
+    .await
     {
-        info!(context, "failure: {}", err);
-        Err(ConfigurationError {
-            config: inf,
-            msg: err.to_string(),
-        })
-    } else {
-        info!(context, "success: {}", inf);
-        Ok(())
+        Err(err) => {
+            info!(context, "failure: {}", err);
+            return Err(ConfigurationError {
+                config: inf,
+                msg: err.to_string(),
+            });
+        }
+        Ok(imap) => imap,
+    };
+
+    match imap.connect(context).await {
+        Err(err) => {
+            info!(context, "failure: {}", err);
+            Err(ConfigurationError {
+                config: inf,
+                msg: err.to_string(),
+            })
+        }
+        Ok(()) => {
+            info!(context, "success: {}", inf);
+            Ok(imap)
+        }
     }
 }
 
 async fn try_smtp_one_param(
     context: &Context,
     param: &ServerLoginParam,
+    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
+        param.user,
+        param.server,
+        param.port,
+        param.security,
+        param.certificate_checks,
+        oauth2,
+        if let Some(socks5_config) = socks5_config {
+            socks5_config.to_string()
+        } else {
+            "None".to_string()
+        }
     );
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .connect(
+            context,
+            param,
+            socks5_config,
+            addr,
+            oauth2,
+            provider_strict_tls,
+        )
         .await
     {
         info!(context, "failure: {}", err);
@@ -616,10 +669,10 @@ pub enum Error {
     },
 
     #[error("Failed to get URL: {0}")]
-    ReadUrlError(#[from] self::read_url::Error),
+    ReadUrl(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]
-    RedirectionError,
+    Redirection,
 }
 
 #[cfg(test)]

src/configure/auto_mozilla.rs

@@ -1,6 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
+//! Documentation: <https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
 use quick_xml::events::{BytesStart, Event};
 
 use std::io::BufRead;
@@ -251,10 +251,10 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
-    url: impl AsRef<str>,
+    url: &str,
     param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
-    let xml_raw = read_url(context, url.as_ref()).await?;
+    let xml_raw = read_url(context, url).await?;
 
     let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -15,27 +15,27 @@ use super::{Error, ServerParams};
 
 /// Result of parsing a single `Protocol` tag.
 ///
-/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
+/// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox>
 #[derive(Debug)]
 struct ProtocolTag {
     /// Server type, such as "IMAP", "SMTP" or "POP3".
     ///
-    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
+    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox>
     pub typ: String,
 
     /// Server identifier, hostname or IP address for IMAP and SMTP.
     ///
-    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
+    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox>
     pub server: String,
 
     /// Network port.
     ///
-    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
+    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox>
     pub port: u16,
 
     /// Whether connection should be secure, "on" or "off", default is "on".
     ///
-    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
+    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox>
     pub ssl: bool,
 }
 
@@ -203,7 +203,7 @@ pub(crate) async fn outlk_autodiscover(
             }
         }
     }
-    Err(Error::RedirectionError)
+    Err(Error::Redirection)
 }
 
 #[cfg(test)]

src/configure/server_params.rs

@@ -25,16 +25,20 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
-            self.username = addr.to_string();
-            res.push(self.clone());
+            res.push(Self {
+                username: addr.to_string(),
+                ..self.clone()
+            });
 
             if let Some(at) = addr.find('@') {
-                self.username = addr.split_at(at).0.to_string();
-                res.push(self);
+                res.push(Self {
+                    username: addr.split_at(at).0.to_string(),
+                    ..self
+                });
             }
         } else {
             res.push(self)
@@ -42,24 +46,28 @@ impl ServerParams {
         res
     }
 
-    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
-        let mut res = Vec::new();
+    fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
         if self.hostname.is_empty() {
-            self.hostname = param_domain.to_string();
-            res.push(self.clone());
-
-            self.hostname = match self.protocol {
-                Protocol::Imap => "imap.".to_string() + param_domain,
-                Protocol::Smtp => "smtp.".to_string() + param_domain,
-            };
-            res.push(self.clone());
-
-            self.hostname = "mail.".to_string() + param_domain;
-            res.push(self);
+            vec![
+                Self {
+                    hostname: param_domain.to_string(),
+                    ..self.clone()
+                },
+                Self {
+                    hostname: match self.protocol {
+                        Protocol::Imap => "imap.".to_string() + param_domain,
+                        Protocol::Smtp => "smtp.".to_string() + param_domain,
+                    },
+                    ..self.clone()
+                },
+                Self {
+                    hostname: "mail.".to_string() + param_domain,
+                    ..self
+                },
+            ]
         } else {
-            res.push(self);
+            vec![self]
         }
-        res
     }
 
     fn expand_ports(mut self) -> Vec<ServerParams> {
@@ -78,39 +86,47 @@ impl ServerParams {
             }
         }
 
-        let mut res = Vec::new();
         if self.port == 0 {
             // Neither port nor security is set.
             //
             // Try common secure combinations.
 
-            // Try STARTTLS
-            self.socket = Socket::Starttls;
-            self.port = match self.protocol {
-                Protocol::Imap => 143,
-                Protocol::Smtp => 587,
-            };
-            res.push(self.clone());
-
-            // Try TLS
-            self.socket = Socket::Ssl;
-            self.port = match self.protocol {
-                Protocol::Imap => 993,
-                Protocol::Smtp => 465,
-            };
-            res.push(self);
+            vec![
+                // Try STARTTLS
+                Self {
+                    socket: Socket::Starttls,
+                    port: match self.protocol {
+                        Protocol::Imap => 143,
+                        Protocol::Smtp => 587,
+                    },
+                    ..self.clone()
+                },
+                // Try TLS
+                Self {
+                    socket: Socket::Ssl,
+                    port: match self.protocol {
+                        Protocol::Imap => 993,
+                        Protocol::Smtp => 465,
+                    },
+                    ..self
+                },
+            ]
         } else if self.socket == Socket::Automatic {
-            // Try TLS over user-provided port.
-            self.socket = Socket::Ssl;
-            res.push(self.clone());
-
-            // Try STARTTLS over user-provided port.
-            self.socket = Socket::Starttls;
-            res.push(self);
+            vec![
+                // Try TLS over user-provided port.
+                Self {
+                    socket: Socket::Ssl,
+                    ..self.clone()
+                },
+                // Try STARTTLS over user-provided port.
+                Self {
+                    socket: Socket::Starttls,
+                    ..self
+                },
+            ]
         } else {
-            res.push(self);
+            vec![self]
         }
-        res
     }
 }
 

src/constants.rs

@@ -1,4 +1,5 @@
-//! # Constants
+//! # Constants.
+use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
@@ -15,15 +16,16 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(i8)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
-    Deaddrop = 2,
+    Request = 2,
 }
 
 impl Default for Blocked {
@@ -32,7 +34,9 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -46,7 +50,9 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -59,7 +65,9 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -73,7 +81,9 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -113,8 +123,6 @@ pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 // do not use too small value that will annoy users checking for nonexistant updates.
 pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
 
-/// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub const DC_CHAT_ID_DEADDROP: ChatId = ChatId::new(1);
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3);
 /// only an indicator in a chatlist
@@ -133,10 +141,11 @@ pub const DC_CHAT_ID_LAST_SPECIAL: ChatId = ChatId::new(9);
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     IntoStaticStr,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(u32)]
 pub enum Chattype {
@@ -156,36 +165,18 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// string that indicates sth. is left out or truncated
-pub const DC_ELLIPSE: &str = "[...]";
+/// String that indicates that something is left out or truncated.
+pub const DC_ELLIPSIS: &str = "[...]";
 
-/// to keep bubbles and chat flow usable,
-/// and to avoid problems with controls using very long texts,
-/// we limit the text length to DC_DESIRED_TEXT_LEN.
-/// if the text is longer, the full text can be retrieved usind has_html()/get_html().
+/// Message length limit.
 ///
-/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
-/// (a bit less as truncation may not be exact and ellipses may be added).
+/// To keep bubbles and chat flow usable and to avoid problems with controls using very long texts,
+/// we limit the text length to `DC_DESIRED_TEXT_LEN`.  If the text is longer, the full text can be
+/// retrieved using has_html()/get_html().
 ///
-/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
-/// define max. number of bytes, _not_ unicode graphemes.
-/// in general, that seems to be okay for such an upper limit,
-/// esp. as calculating the number of graphemes is not simple
-/// (one graphemes may be a sequence of code points which is a sequence of bytes).
-/// also even if we have the exact number of graphemes,
-/// that would not always help on getting an idea about the screen space used
-/// (to keep bubbles and chat flow usable).
-///
-/// therefore, the number of bytes is only a very rough estimation,
-/// however, the ~30K seems to work okayish for a while,
-/// if it turns out, it is too few for some alphabet, we can still increase.
-pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
-
-/// approx. max. length (number of bytes) returned by dc_msg_get_text()
-pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
-
-/// approx. max. length returned by dc_get_msg_info()
-pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
+/// Note that for simplicity maximum length is defined as the number of Unicode Scalar Values (Rust
+/// `char`s), not Unicode Grapheme Clusters.
+pub const DC_DESIRED_TEXT_LEN: usize = 5000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -247,9 +238,10 @@ pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(u32)]
 pub enum Viewtype {
@@ -392,7 +384,7 @@ mod tests {
         assert_eq!(Blocked::Not, Blocked::default());
         assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
         assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
-        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
+        assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
     }
 
     #[test]

src/context.rs

@@ -1,4 +1,4 @@
-//! Context module
+//! Context module.
 
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
@@ -6,14 +6,12 @@ use std::ops::Deref;
 use std::time::{Instant, SystemTime};
 
 use anyhow::{bail, ensure, Result};
-use async_std::prelude::*;
 use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
     task,
 };
-use sqlx::Row;
 
 use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
@@ -24,6 +22,7 @@ use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
+use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
@@ -64,6 +63,10 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
+    /// Recently loaded quota information, if any.
+    /// Set to `None` if quota was never tried to load.
+    pub(crate) quota: RwLock<Option<QuotaInfo>>,
+
     pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
 
     /// ID for this `Context` in the current process.
@@ -91,7 +94,7 @@ pub struct RunningState {
 pub fn get_info() -> BTreeMap<&'static str, String> {
     let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
-    res.insert("sqlite_version", crate::sql::version().to_string());
+    res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("num_cpus", num_cpus::get().to_string());
     res.insert("level", "awesome".into());
@@ -141,6 +144,7 @@ impl Context {
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
+            quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
         };
@@ -163,7 +167,9 @@ impl Context {
 
         {
             let l = &mut *self.inner.scheduler.write().await;
-            l.start(self.clone()).await;
+            if let Err(err) = l.start(self.clone()).await {
+                error!(self, "Failed to start IO: {}", err)
+            }
         }
     }
 
@@ -279,10 +285,11 @@ impl Context {
         let l2 = LoginParam::from_database(self, "configured_").await?;
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
-        let real_msgs = message::get_real_msg_cnt(self).await as usize;
-        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
+        let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
+        let request_msgs = message::get_request_msg_cnt(self).await as usize;
         let contacts = Contact::get_real_cnt(self).await? as usize;
         let is_configured = self.get_config_int(Config::Configured).await?;
+        let socks5_enabled = self.get_config_int(Config::Socks5Enabled).await?;
         let dbversion = self
             .sql
             .get_raw_config_int("dbversion")
@@ -290,7 +297,7 @@ impl Context {
             .unwrap_or_default();
         let journal_mode = self
             .sql
-            .query_get_value(sqlx::query("PRAGMA journal_mode;"))
+            .query_get_value("PRAGMA journal_mode;", paramsv![])
             .await?
             .unwrap_or_else(|| "unknown".to_string());
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
@@ -299,12 +306,12 @@ impl Context {
 
         let prv_key_cnt = self
             .sql
-            .count(sqlx::query("SELECT COUNT(*) FROM keypairs;"))
+            .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
             .await?;
 
         let pub_key_cnt = self
             .sql
-            .count(sqlx::query("SELECT COUNT(*) FROM acpeerstates;"))
+            .count("SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await?;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
             Ok(key) => key.fingerprint().hex(),
@@ -336,8 +343,8 @@ impl Context {
         // insert values
         res.insert("bot", self.get_config_int(Config::Bot).await?.to_string());
         res.insert("number_of_chats", chats.to_string());
-        res.insert("number_of_chat_messages", real_msgs.to_string());
-        res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
+        res.insert("number_of_chat_messages", unblocked_msgs.to_string());
+        res.insert("messages_in_contact_requests", request_msgs.to_string());
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
@@ -351,6 +358,7 @@ impl Context {
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
+        res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
         res.insert(
@@ -415,6 +423,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "quota_exceeding",
+            self.get_config_int(Config::QuotaExceeding)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -422,7 +436,7 @@ impl Context {
         Ok(res)
     }
 
-    /// Get a list of fresh, unmuted messages in any chat but deaddrop.
+    /// Get a list of fresh, unmuted messages in unblocked chats.
     ///
     /// The list starts with the most recent message
     /// and is typically used to show notifications.
@@ -431,8 +445,8 @@ impl Context {
     pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
         let list = self
             .sql
-            .fetch(
-                sqlx::query(concat!(
+            .query_map(
+                concat!(
                     "SELECT m.id",
                     " FROM msgs m",
                     " LEFT JOIN contacts ct",
@@ -446,13 +460,17 @@ impl Context {
                     "   AND c.blocked=0",
                     "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
-                ))
-                .bind(MessageState::InFresh)
-                .bind(time()),
+                ),
+                paramsv![MessageState::InFresh, time()],
+                |row| row.get::<_, MsgId>(0),
+                |rows| {
+                    let mut list = Vec::new();
+                    for row in rows {
+                        list.push(row?);
+                    }
+                    Ok(list)
+                },
             )
-            .await?
-            .map(|row| row?.try_get("id"))
-            .collect::<sqlx::Result<_>>()
             .await?;
         Ok(list)
     }
@@ -461,22 +479,31 @@ impl Context {
     ///
     /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
     /// is `None` this searches messages from all chats.
-    pub async fn search_msgs(
-        &self,
-        chat_id: Option<ChatId>,
-        query: impl AsRef<str>,
-    ) -> Result<Vec<MsgId>> {
-        let real_query = query.as_ref().trim();
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: &str) -> Result<Vec<MsgId>> {
+        let real_query = query.trim();
         if real_query.is_empty() {
             return Ok(Vec::new());
         }
         let str_like_in_text = format!("%{}%", real_query);
 
+        let do_query = |query, params| {
+            self.sql.query_map(
+                query,
+                params,
+                |row| row.get::<_, MsgId>("id"),
+                |rows| {
+                    let mut ret = Vec::new();
+                    for id in rows {
+                        ret.push(id?);
+                    }
+                    Ok(ret)
+                },
+            )
+        };
+
         let list = if let Some(chat_id) = chat_id {
-            self.sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT m.id AS id, m.timestamp AS timestamp
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -485,18 +512,9 @@ impl Context {
                    AND ct.blocked=0
                    AND txt LIKE ?
                  ORDER BY m.timestamp,m.id;",
-                    )
-                    .bind(chat_id)
-                    .bind(str_like_in_text),
-                )
-                .await?
-                .map(|row| {
-                    let row = row?;
-                    let id = row.try_get::<MsgId, _>("id")?;
-                    Ok(id)
-                })
-                .collect::<sqlx::Result<Vec<MsgId>>>()
-                .await?
+                paramsv![chat_id, str_like_in_text],
+            )
+            .await?
         } else {
             // For performance reasons results are sorted only by `id`, that is in the order of
             // message reception.
@@ -508,10 +526,8 @@ impl Context {
             // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
             // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
             // The limit is documented and UI may add a hint when getting 1000 results.
-            self.sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT m.id AS id, m.timestamp AS timestamp
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -523,44 +539,32 @@ impl Context {
                    AND ct.blocked=0
                    AND m.txt LIKE ?
                  ORDER BY m.id DESC LIMIT 1000",
-                    )
-                    .bind(str_like_in_text),
-                )
-                .await?
-                .map(|row| {
-                    let row = row?;
-                    let id = row.try_get::<MsgId, _>("id")?;
-                    Ok(id)
-                })
-                .collect::<sqlx::Result<Vec<MsgId>>>()
-                .await?
+                paramsv![str_like_in_text],
+            )
+            .await?
         };
 
         Ok(list)
     }
 
-    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_inbox(&self, folder_name: &str) -> Result<bool> {
         let inbox = self.get_config(Config::ConfiguredInboxFolder).await?;
-        Ok(inbox == Some(folder_name.as_ref().to_string()))
+        Ok(inbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_sentbox(&self, folder_name: &str) -> Result<bool> {
         let sentbox = self.get_config(Config::ConfiguredSentboxFolder).await?;
-
-        Ok(sentbox == Some(folder_name.as_ref().to_string()))
+        Ok(sentbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
         let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
-
-        Ok(mvbox == Some(folder_name.as_ref().to_string()))
+        Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_spam_folder(&self, folder_name: impl AsRef<str>) -> Result<bool> {
-        let is_spam = self.get_config(Config::ConfiguredSpamFolder).await?
-            == Some(folder_name.as_ref().to_string());
-
-        Ok(is_spam)
+    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
+        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
+        Ok(spam.as_deref() == Some(folder_name))
     }
 
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
@@ -569,6 +573,13 @@ impl Context {
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
+
+    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
+        let mut wal_fname = OsString::new();
+        wal_fname.push(dbfile.file_name().unwrap_or_default());
+        wal_fname.push("-wal");
+        dbfile.with_file_name(wal_fname)
+    }
 }
 
 impl InnerContext {
@@ -613,8 +624,7 @@ mod tests {
     use super::*;
 
     use crate::chat::{
-        create_by_contact_id, get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat,
-        MuteDuration,
+        get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
     };
     use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
@@ -747,9 +757,8 @@ mod tests {
         // we need to modify the database directly
         t.sql
             .execute(
-                sqlx::query("UPDATE chats SET muted_until=? WHERE id=?;")
-                    .bind(time() - 3600)
-                    .bind(bob.id),
+                "UPDATE chats SET muted_until=? WHERE id=?;",
+                paramsv![time() - 3600, bob.id],
             )
             .await
             .unwrap();
@@ -766,7 +775,10 @@ mod tests {
         // to test get_fresh_msgs() with invalid mute_until (everything < -1),
         // that results in "muted forever" by definition.
         t.sql
-            .execute(sqlx::query("UPDATE chats SET muted_until=-2 WHERE id=?;").bind(bob.id))
+            .execute(
+                "UPDATE chats SET muted_until=-2 WHERE id=?;",
+                paramsv![bob.id],
+            )
             .await
             .unwrap();
         let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
@@ -874,6 +886,10 @@ mod tests {
             "send_security",
             "server_flags",
             "smtp_certificate_checks",
+            "socks5_host",
+            "socks5_port",
+            "socks5_user",
+            "socks5_password",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -895,7 +911,7 @@ mod tests {
     #[async_std::test]
     async fn test_search_msgs() -> Result<()> {
         let alice = TestContext::new_alice().await;
-        let self_talk = create_by_contact_id(&alice, DC_CONTACT_ID_SELF).await?;
+        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
         let chat = alice
             .create_chat_with_contact("Bob", "bob@example.org")
             .await;

src/dc_tools.rs

@@ -17,7 +17,7 @@ use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::message::Message;
@@ -29,7 +29,7 @@ use crate::stock_str;
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let count = buf.chars().count();
-    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
+    if count > approx_chars + DC_ELLIPSIS.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -37,9 +37,9 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSIS))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSIS))
         }
     } else {
         Cow::Borrowed(buf)
@@ -608,19 +608,8 @@ impl FromStr for EmailAddress {
                 if local.is_empty() {
                     return err("empty string is not valid for local part");
                 }
-                if domain.len() <= 3 {
-                    return err("domain is too short");
-                }
-                let dot = domain.find('.');
-                match dot {
-                    None => {
-                        return err("invalid domain");
-                    }
-                    Some(dot_idx) => {
-                        if dot_idx >= domain.len() - 2 {
-                            return err("invalid domain");
-                        }
-                    }
+                if domain.is_empty() {
+                    return err("missing domain after '@'");
                 }
                 Ok(EmailAddress {
                     local: (*local).to_string(),
@@ -632,10 +621,17 @@ impl FromStr for EmailAddress {
     }
 }
 
+impl rusqlite::types::ToSql for EmailAddress {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Text(self.to_string());
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
-pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .as_ref()
         .replace("\n", " ")
         .replace("\r", " ")
         .trim()
@@ -659,7 +655,7 @@ pub fn remove_subject_prefix(last_subject: &str) -> String {
         0
     } else {
         // "Antw:" is the longest abbreviation in
-        // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+        // <https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages>,
         // so look at the first _5_ characters:
         match last_subject.chars().take(5).position(|c| c == ':') {
             Some(prefix_end) => prefix_end + 1,
@@ -715,10 +711,7 @@ mod tests {
         assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
 
         assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
-        assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
-            "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
-        );
+        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
 
         // 9 characters, so no truncation
         assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
@@ -818,12 +811,19 @@ mod tests {
                 domain: "domain.tld".into(),
             }
         );
+        assert_eq!(
+            "user@localhost".parse::<EmailAddress>().unwrap(),
+            EmailAddress {
+                local: "user".into(),
+                domain: "localhost".into()
+            }
+        );
         assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
         assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
+        assert!("tt.dd@uu".parse::<EmailAddress>().is_ok());
+        assert!("u@d".parse::<EmailAddress>().is_ok());
+        assert!("u@d.".parse::<EmailAddress>().is_ok());
+        assert!("u@d.t".parse::<EmailAddress>().is_ok());
         assert_eq!(
             "u@d.tt".parse::<EmailAddress>().unwrap(),
             EmailAddress {
@@ -831,7 +831,7 @@ mod tests {
                 domain: "d.tt".into(),
             }
         );
-        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
+        assert!("u@tt".parse::<EmailAddress>().is_ok());
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 

src/dehtml.rs

@@ -1,4 +1,4 @@
-//! De-HTML
+//! De-HTML.
 //!
 //! A module to remove HTML tags from the email text
 

src/e2ee.rs

@@ -178,7 +178,9 @@ pub async fn try_decrypt(
     let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
-        peerstate.handle_fingerprint_change(context).await?;
+        peerstate
+            .handle_fingerprint_change(context, message_time)
+            .await?;
         if let Some(key) = &peerstate.public_key {
             public_keyring_for_validate.add(key.clone());
         } else if let Some(key) = &peerstate.gossip_key {
@@ -333,7 +335,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     false
 }
 
-/// Check if a MIME structure contains a multipart/report part.
+/// Checks if a MIME structure contains a multipart/report part.
 ///
 /// As reports are often unencrypted, we do not reset the Autocrypt header in
 /// this case.

src/ephemeral.rs

@@ -1,4 +1,4 @@
-//! # Ephemeral messages
+//! # Ephemeral messages.
 //!
 //! Ephemeral messages are messages that have an Ephemeral-Timer
 //! header attached to them, which specifies time in seconds after
@@ -61,25 +61,21 @@ use std::num::ParseIntError;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
-use anyhow::{ensure, Context as _, Error};
+use anyhow::{ensure, Context as _, Result};
 use async_std::task;
 use serde::{Deserialize, Serialize};
-use sqlx::Row;
 
+use crate::chat::{send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::events::EventType;
+use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::sql;
 use crate::stock_str;
-use crate::{
-    chat::{lookup_by_contact_id, send_msg, ChatId},
-    job,
-};
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -124,51 +120,39 @@ impl FromStr for Timer {
     }
 }
 
-impl sqlx::Type<sqlx::Sqlite> for Timer {
-    fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
-        <i64 as sqlx::Type<_>>::type_info()
-    }
-
-    fn compatible(ty: &sqlx::sqlite::SqliteTypeInfo) -> bool {
-        <i64 as sqlx::Type<_>>::compatible(ty)
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
     }
 }
 
-impl<'q> sqlx::Encode<'q, sqlx::Sqlite> for Timer {
-    fn encode_by_ref(
-        &self,
-        args: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>,
-    ) -> sqlx::encode::IsNull {
-        args.push(sqlx::sqlite::SqliteArgumentValue::Int64(
-            self.to_u32() as i64
-        ));
-
-        sqlx::encode::IsNull::No
-    }
-}
-
-impl<'r> sqlx::Decode<'r, sqlx::Sqlite> for Timer {
-    fn decode(value: sqlx::sqlite::SqliteValueRef<'r>) -> Result<Self, sqlx::error::BoxDynError> {
-        let value: i64 = sqlx::Decode::decode(value)?;
-        if value == 0 {
-            Ok(Self::Disabled)
-        } else if let Ok(duration) = u32::try_from(value) {
-            Ok(Self::Enabled { duration })
-        } else {
-            Err(Box::new(sqlx::Error::Decode(Box::new(
-                crate::error::OutOfRangeError,
-            ))))
-        }
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
     }
 }
 
 impl ChatId {
     /// Get ephemeral message timer value in seconds.
-    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer> {
         let timer = context
             .sql
             .query_get_value(
-                sqlx::query("SELECT ephemeral_timer FROM chats WHERE id=?;").bind(self),
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
             )
             .await?;
         Ok(timer.unwrap_or_default())
@@ -182,19 +166,16 @@ impl ChatId {
         self,
         context: &Context,
         timer: Timer,
-    ) -> Result<(), Error> {
+    ) -> Result<()> {
         ensure!(!self.is_special(), "Invalid chat ID");
 
         context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE chats
+                "UPDATE chats
              SET ephemeral_timer=?
              WHERE id=?;",
-                )
-                .bind(timer)
-                .bind(self),
+                paramsv![timer, self],
             )
             .await?;
 
@@ -208,7 +189,7 @@ impl ChatId {
     /// Set ephemeral message timer value in seconds.
     ///
     /// If timer value is 0, disable ephemeral message timer.
-    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<()> {
         if timer == self.get_ephemeral_timer(context).await? {
             return Ok(());
         }
@@ -233,45 +214,44 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     from_id: u32,
 ) -> String {
     match timer {
-        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id as u32).await,
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
             0..=59 => {
-                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id as u32)
-                    .await
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
             }
-            60 => stock_str::msg_ephemeral_timer_minute(context, from_id as u32).await,
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
             61..=3599 => {
                 stock_str::msg_ephemeral_timer_minutes(
                     context,
                     format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id as u32).await,
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
             3601..=86399 => {
                 stock_str::msg_ephemeral_timer_hours(
                     context,
                     format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            86400 => stock_str::msg_ephemeral_timer_day(context, from_id as u32).await,
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
             86401..=604_799 => {
                 stock_str::msg_ephemeral_timer_days(
                     context,
                     format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id as u32).await,
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
             _ => {
                 stock_str::msg_ephemeral_timer_weeks(
                     context,
                     format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
@@ -284,15 +264,14 @@ impl MsgId {
     pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
         let res = match context
             .sql
-            .query_get_value::<_, i64>(
-                sqlx::query("SELECT ephemeral_timer FROM msgs WHERE id=?").bind(self),
+            .query_get_value(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
             )
             .await?
         {
             None | Some(0) => Timer::Disabled,
-            Some(duration) => Timer::Enabled {
-                duration: u32::try_from(duration)?,
-            },
+            Some(duration) => Timer::Enabled { duration },
         };
         Ok(res)
     }
@@ -305,14 +284,10 @@ impl MsgId {
             context
                 .sql
                 .execute(
-                    sqlx::query(
-                        "UPDATE msgs SET ephemeral_timestamp = ? \
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
                 WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
                 AND id = ?",
-                    )
-                    .bind(ephemeral_timestamp)
-                    .bind(ephemeral_timestamp)
-                    .bind(self),
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
             schedule_ephemeral_task(context).await;
@@ -329,14 +304,13 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
     let mut updated = context
         .sql
         .execute(
-            sqlx::query(
-                // If you change which information is removed here, also change MsgId::trash() and
-                // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-                r#"
+            // If you change which information is removed here, also change MsgId::trash() and
+            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
+            r#"
 UPDATE msgs
 SET 
   chat_id=?, txt='', subject='', txt_raw='', 
@@ -346,24 +320,19 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            )
-            .bind(DC_CHAT_ID_TRASH)
-            .bind(time())
-            .bind(DC_CHAT_ID_TRASH),
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
+        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            .await?
+            .unwrap_or_default();
 
         let threshold_timestamp = time() - delete_device_after;
 
@@ -374,19 +343,19 @@ WHERE
         let rows_modified = context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE msgs \
+                "UPDATE msgs \
              SET txt = 'DELETED', chat_id = ? \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
              AND chat_id != ?",
-                )
-                .bind(DC_CHAT_ID_TRASH)
-                .bind(threshold_timestamp)
-                .bind(DC_CHAT_ID_LAST_SPECIAL)
-                .bind(self_chat_id)
-                .bind(device_chat_id),
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
             )
             .await
             .context("deleted update failed")?;
@@ -412,8 +381,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
-            sqlx::query(
-                r#"
+            r#"
     SELECT ephemeral_timestamp
     FROM msgs
     WHERE ephemeral_timestamp != 0
@@ -421,8 +389,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
     ORDER BY ephemeral_timestamp ASC
     LIMIT 1;
     "#,
-            )
-            .bind(DC_CHAT_ID_TRASH), // Trash contains already deleted messages, skip them
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
@@ -475,7 +442,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 ///
 /// It looks up the trash chat too, to find messages that are already
 /// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
     let threshold_timestamp = match context.get_config_delete_server_after().await? {
@@ -483,11 +450,10 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
         Some(delete_server_after) => now - delete_server_after,
     };
 
-    let row = context
+    context
         .sql
-        .fetch_optional(
-            sqlx::query(
-                "SELECT id FROM msgs \
+        .query_row_optional(
+            "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
@@ -495,19 +461,13 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
          AND server_uid != 0 \
          AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            )
-            .bind(threshold_timestamp)
-            .bind(now)
-            .bind(job::Action::DeleteMsgOnImap),
+            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                Ok(msg_id)
+            },
         )
-        .await?;
-
-    if let Some(row) = row {
-        let msg_id = row.try_get(0)?;
-        Ok(Some(msg_id))
-    } else {
-        Ok(None)
-    }
+        .await
 }
 
 /// Start ephemeral timers for seen messages if they are not started
@@ -519,21 +479,21 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
 ///
 /// This function is supposed to be called in the background,
 /// e.g. from housekeeping task.
-pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
     context
         .sql
         .execute(
-            sqlx::query(
-                "UPDATE msgs \
+            "UPDATE msgs \
     SET ephemeral_timestamp = ? + ephemeral_timer \
     WHERE ephemeral_timer > 0 \
     AND ephemeral_timestamp = 0 \
     AND state NOT IN (?, ?, ?)",
-            )
-            .bind(time())
-            .bind(MessageState::InFresh)
-            .bind(MessageState::InNoticed)
-            .bind(MessageState::OutDraft),
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
         )
         .await?;
 
@@ -770,7 +730,10 @@ mod tests {
         // Check that the msg will be deleted on the server
         // First of all, set a server_uid so that DC thinks that it's actually possible to delete
         t.sql
-            .execute(sqlx::query("UPDATE msgs SET server_uid=1 WHERE id=?").bind(msg.sender_msg_id))
+            .execute(
+                "UPDATE msgs SET server_uid=1 WHERE id=?",
+                paramsv![msg.sender_msg_id],
+            )
             .await
             .unwrap();
         let job = job::load_imap_deletion_job(&t).await.unwrap();
@@ -808,7 +771,7 @@ mod tests {
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
-                .query_get_value(sqlx::query("SELECT txt_raw FROM msgs WHERE id=?;").bind(msg_id))
+                .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
                 .await
                 .unwrap();
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);

src/error.rs

@@ -1,9 +1,5 @@
 //! # Error handling
 
-#[derive(Debug, thiserror::Error)]
-#[error("Out of Range")]
-pub struct OutOfRangeError;
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -1,4 +1,4 @@
-//! # Events specification
+//! # Events specification.
 
 use std::ops::Deref;
 
@@ -62,6 +62,10 @@ impl Events {
 pub struct EventEmitter(Receiver<Event>);
 
 impl EventEmitter {
+    pub(crate) fn into_inner(self) -> Receiver<Event> {
+        self.0
+    }
+
     /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
     pub fn recv_sync(&self) -> Option<Event> {
         async_std::task::block_on(self.recv())
@@ -185,21 +189,6 @@ pub enum EventType {
     #[strum(props(id = "400"))]
     Error(String),
 
-    /// An action cannot be performed because there is no network available.
-    ///
-    /// The library will typically try over after a some time
-    /// and when dc_maybe_network() is called.
-    ///
-    /// Network errors should be reported to users in a non-disturbing way,
-    /// however, as network errors may come in a sequence,
-    /// it is not useful to raise each an every error to the user.
-    ///
-    /// Moreover, if the UI detects that the device is offline,
-    /// it is probably more useful to report this to the user
-    /// instead of the string from data2.
-    #[strum(props(id = "401"))]
-    ErrorNetwork(String),
-
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
@@ -213,6 +202,8 @@ pub enum EventType {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
+    ///
+    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
@@ -328,4 +319,11 @@ pub enum EventType {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+
+    /// The connectivity to the server changed.
+    /// This means that you should refresh the connectivity view
+    /// and possibly the connectivtiy HTML; see dc_get_connectivity() and
+    /// dc_get_connectivity_html() for details.
+    #[strum(props(id = "2100"))]
+    ConnectivityChanged,
 }

src/format_flowed.rs

@@ -1,13 +1,13 @@
-///! # format=flowed support
-///!
-///! Format=flowed is defined in
-///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
-///!
-///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
-///! during formatting, i.e., DelSp parameter introduced in RFC 3676
-///! is assumed to be set to "no".
-///!
-///! For received messages, DelSp parameter is honoured.
+//! # format=flowed support.
+//!
+//! Format=flowed is defined in
+//! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+//!
+//! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+//! during formatting, i.e., DelSp parameter introduced in RFC 3676
+//! is assumed to be set to "no".
+//!
+//! For received messages, DelSp parameter is honoured.
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///
@@ -104,13 +104,13 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
 
     for line in text.split('\n') {
         // Revert space-stuffing
-        let line = line.strip_prefix(" ").unwrap_or(line);
+        let line = line.strip_prefix(' ').unwrap_or(line);
 
         if !skip_newline {
             result.push('\n');
         }
 
-        if let Some(line) = line.strip_suffix(" ") {
+        if let Some(line) = line.strip_suffix(' ') {
             // Flowed line
             result += line;
             if !delsp {

src/headerdef.rs

@@ -1,3 +1,5 @@
+//! # List of email headers.
+
 use crate::strum::AsStaticRef;
 use mailparse::{MailHeader, MailHeaderMap};
 
@@ -25,6 +27,12 @@ pub enum HeaderDef {
     /// we need to check that header as well.
     XMicrosoftOriginalMessageId,
 
+    /// Thunderbird header used to store Draft information.
+    ///
+    /// Thunderbird 78.11.0 does not set \Draft flag on messages saved as "Template", but sets this
+    /// header, so it can be used to ignore such messages.
+    XMozillaDraftInfo,
+
     ListId,
     References,
     InReplyTo,

src/html.rs

@@ -1,11 +1,12 @@
-///! # Get message as HTML.
-///!
-///! Use `Message.has_html()` to check if the UI shall render a
-///! corresponding button and `MsgId.get_html()` to get the full message.
-///!
-///! Even when the original mime-message is not HTML,
-///! `MsgId.get_html()` will return HTML -
-///! this allows nice quoting, handling linebreaks properly etc.
+//! # Get message as HTML.
+//!
+//! Use `Message.has_html()` to check if the UI shall render a
+//! corresponding button and `MsgId.get_html()` to get the full message.
+//!
+//! Even when the original mime-message is not HTML,
+//! `MsgId.get_html()` will return HTML -
+//! this allows nice quoting, handling linebreaks properly etc.
+
 use futures::future::FutureExt;
 use std::future::Future;
 use std::pin::Pin;
@@ -248,7 +249,7 @@ impl MsgId {
         let rawmime = message::get_mime_headers(context, self).await?;
 
         if !rawmime.is_empty() {
-            match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
+            match HtmlMsgParser::from_bytes(context, &rawmime).await {
                 Err(err) => {
                     warn!(context, "get_html: parser error: {}", err);
                     Ok(None)
@@ -424,10 +425,10 @@ test some special html-characters as < > and & but also " and &#x
     }
 
     #[async_std::test]
-    async fn test_get_html_empty() {
+    async fn test_get_html_invalid_msgid() {
         let t = TestContext::new().await;
-        let msg_id = MsgId::new_unset();
-        assert!(msg_id.get_html(&t).await.unwrap().is_none())
+        let msg_id = MsgId::new(100);
+        assert!(msg_id.get_html(&t).await.is_err())
     }
 
     #[async_std::test]
@@ -550,4 +551,26 @@ test some special html-characters as < > and & but also " and &#x
         let html = msg.get_id().get_html(&bob).await.unwrap().unwrap();
         assert!(html.contains("<b>html</b> text"));
     }
+
+    #[async_std::test]
+    async fn test_cp1252_html() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        t.set_config(Config::ShowEmails, Some("2")).await?;
+        dc_receive_imf(
+            &t,
+            include_bytes!("../test-data/message/cp1252-html.eml"),
+            "INBOX",
+            0,
+            false,
+        )
+        .await?;
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.viewtype, Viewtype::Text);
+        assert!(msg.text.as_ref().unwrap().contains("foo bar ä ö ü ß"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&t).await?.unwrap();
+        println!("{}", html);
+        assert!(html.contains("foo bar ä ö ü ß"));
+        Ok(())
+    }
 }

src/imap/client.rs

@@ -1,16 +1,23 @@
-use std::ops::{Deref, DerefMut};
-
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
+use std::{
+    ops::{Deref, DerefMut},
+    time::Duration,
 };
+
+use anyhow::{Context as _, Result};
+
+use async_imap::Client as ImapClient;
+
+use async_smtp::ServerAddress;
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::dc_build_tls;
+use crate::login_param::{dc_build_tls, Socks5Config};
 
 use super::session::SessionStream;
 
+/// IMAP write and read timeout in seconds.
+const IMAP_TIMEOUT: u64 = 30;
+
 #[derive(Debug)]
 pub(crate) struct Client {
     is_secure: bool,
@@ -32,64 +39,42 @@ impl DerefMut for Client {
 }
 
 impl Client {
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
+    pub async fn login(self, username: &str, password: &str) -> Result<Session> {
+        let Client { inner, .. } = self;
         let session = inner
             .login(username, password)
             .await
-            .map_err(|(err, client)| {
-                (
-                    err,
-                    Client {
-                        is_secure,
-                        inner: client,
-                    },
-                )
-            })?;
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+    pub async fn authenticate(
         self,
-        auth_type: S,
-        authenticator: A,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
-        let session =
-            inner
-                .authenticate(auth_type, authenticator)
-                .await
-                .map_err(|(err, client)| {
-                    (
-                        err,
-                        Client {
-                            is_secure,
-                            inner: client,
-                        },
-                    )
-                })?;
+        auth_type: &str,
+        authenticator: impl async_imap::Authenticator,
+    ) -> Result<Session> {
+        let Client { inner, .. } = self;
+        let session = inner
+            .authenticate(auth_type, authenticator)
+            .await
+            .map_err(|(err, _client)| err)?;
         Ok(Session { inner: session })
     }
 
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
+    pub async fn connect_secure(
+        addr: impl net::ToSocketAddrs,
+        domain: &str,
         strict_tls: bool,
-    ) -> ImapResult<Self> {
+    ) -> Result<Self> {
         let stream = TcpStream::connect(addr).await?;
         let tls = dc_build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(domain.as_ref(), stream).await?);
+        let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
         let mut client = ImapClient::new(tls_stream);
 
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: true,
@@ -97,14 +82,14 @@ impl Client {
         })
     }
 
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> Result<Self> {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .context("failed to read greeting")?;
 
         Ok(Client {
             is_secure: false,
@@ -112,7 +97,56 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
+    pub async fn connect_secure_socks5(
+        target_addr: &ServerAddress,
+        strict_tls: bool,
+        socks5_config: Socks5Config,
+    ) -> Result<Self> {
+        let socks5_stream: Box<dyn SessionStream> = Box::new(
+            socks5_config
+                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
+                .await?,
+        );
+
+        let tls = dc_build_tls(strict_tls);
+        let tls_stream: Box<dyn SessionStream> =
+            Box::new(tls.connect(target_addr.host.clone(), socks5_stream).await?);
+        let mut client = ImapClient::new(tls_stream);
+
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")?;
+
+        Ok(Client {
+            is_secure: true,
+            inner: client,
+        })
+    }
+
+    pub async fn connect_insecure_socks5(
+        target_addr: &ServerAddress,
+        socks5_config: Socks5Config,
+    ) -> Result<Self> {
+        let socks5_stream: Box<dyn SessionStream> = Box::new(
+            socks5_config
+                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
+                .await?,
+        );
+
+        let mut client = ImapClient::new(socks5_stream);
+        let _greeting = client
+            .read_response()
+            .await
+            .context("failed to read greeting")?;
+
+        Ok(Client {
+            is_secure: false,
+            inner: client,
+        })
+    }
+
+    pub async fn secure(self, domain: &str, strict_tls: bool) -> Result<Client> {
         if self.is_secure {
             Ok(self)
         } else {
@@ -121,7 +155,7 @@ impl Client {
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();
-            let ssl_stream = tls.connect(domain.as_ref(), stream).await?;
+            let ssl_stream = tls.connect(domain, stream).await?;
             let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
 
             Ok(Client {

src/imap/idle.rs

@@ -2,7 +2,6 @@ use super::Imap;
 
 use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
-use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
@@ -25,31 +24,18 @@ impl Imap {
         if !self.can_idle() {
             bail!("IMAP server does not have IDLE capability");
         }
-        self.setup_handle(context).await?;
+        self.prepare(context).await?;
 
-        self.select_folder(context, watch_folder.clone()).await?;
+        self.select_folder(context, watch_folder.as_deref()).await?;
 
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
+        if self.server_sent_unsolicited_exists(context) {
+            return Ok(info);
+        }
+
         if let Some(session) = self.session.take() {
-            // if we have unsolicited responses we directly return
-            let mut unsolicited_exists = false;
-            while let Ok(response) = session.unsolicited_responses.try_recv() {
-                match response {
-                    UnsolicitedResponse::Exists(_) => {
-                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
-                        unsolicited_exists = true;
-                    }
-                    _ => info!(context, "ignoring unsolicited response {:?}", response),
-                }
-            }
-
-            if unsolicited_exists {
-                self.session = Some(session);
-                return Ok(info);
-            }
-
             if let Ok(info) = self.idle_interrupt.try_recv() {
                 info!(context, "skip idle, got interrupt {:?}", info);
                 self.session = Some(session);
@@ -158,7 +144,7 @@ impl Imap {
                     // try to connect with proper login params
                     // (setup_handle_if_needed might not know about them if we
                     // never successfully connected)
-                    if let Err(err) = self.connect_configured(context).await {
+                    if let Err(err) = self.prepare(context).await {
                         warn!(context, "fake_idle: could not connect: {}", err);
                         continue;
                     }
@@ -181,7 +167,7 @@ impl Imap {
                         }
                         Err(err) => {
                             error!(context, "could not fetch from folder: {:#}", err);
-                            self.trigger_reconnect()
+                            self.trigger_reconnect(context).await;
                         }
                     }
                 }

src/imap/scan_folders.rs

@@ -1,13 +1,13 @@
-use std::time::Instant;
+use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
 
-use crate::config::Config;
-use crate::context::Context;
 use crate::imap::Imap;
+use crate::{config::Config, log::LogExt};
+use crate::{context::Context, imap::FolderMeaning};
 use async_std::prelude::*;
 
-use super::{get_folder_meaning, get_folder_meaning_by_name, FolderMeaning};
+use super::{get_folder_meaning, get_folder_meaning_by_name};
 
 impl Imap {
     pub async fn scan_folders(&mut self, context: &Context) -> Result<()> {
@@ -25,14 +25,13 @@ impl Imap {
         }
         info!(context, "Starting full folder scan");
 
-        self.setup_handle(context).await?;
+        self.prepare(context).await?;
         let session = self.session.as_mut();
         let session = session.context("scan_folders(): IMAP No Connection established")?;
         let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
         let watched_folders = get_watched_folders(context).await;
 
-        let mut sentbox_folder = None;
-        let mut spam_folder = None;
+        let mut folder_configs = BTreeMap::new();
 
         for folder in folders {
             let folder = match folder {
@@ -43,45 +42,67 @@ impl Imap {
                 }
             };
 
-            let foldername = folder.name();
-            let folder_meaning = get_folder_meaning(&folder);
-            let folder_name_meaning = get_folder_meaning_by_name(foldername);
-
-            if folder_meaning == FolderMeaning::SentObjects {
-                // Always takes precedent
-                sentbox_folder = Some(folder.name().to_string());
-            } else if folder_meaning == FolderMeaning::Spam {
-                spam_folder = Some(folder.name().to_string());
-            } else if folder_name_meaning == FolderMeaning::SentObjects {
-                // only set iff none has been already set
-                if sentbox_folder.is_none() {
-                    sentbox_folder = Some(folder.name().to_string());
-                }
-            } else if folder_name_meaning == FolderMeaning::Spam && spam_folder.is_none() {
-                spam_folder = Some(folder.name().to_string());
+            // Gmail labels are not folders and should be skipped. For example,
+            // emails appear in the inbox and under "All Mail" as soon as it is
+            // received. The code used to wrongly conclude that the email had
+            // already been moved and left it in the inbox.
+            let folder_name = folder.name();
+            if folder_name.starts_with("[Gmail]") {
+                continue;
             }
 
+            let folder_meaning = get_folder_meaning(&folder);
+            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
+
+            if let Some(config) = folder_meaning.to_config() {
+                // Always takes precedence
+                folder_configs.insert(config, folder.name().to_string());
+            } else if let Some(config) = folder_name_meaning.to_config() {
+                // only set if none has been already set
+                folder_configs
+                    .entry(config)
+                    .or_insert_with(|| folder.name().to_string());
+            }
+
+            let is_drafts = folder_meaning == FolderMeaning::Drafts
+                || (folder_meaning == FolderMeaning::Unknown
+                    && folder_name_meaning == FolderMeaning::Drafts);
+
             // Don't scan folders that are watched anyway
-            if !watched_folders.contains(&foldername.to_string()) {
-                if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
-                    warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
+            if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
+                // Drain leftover unsolicited EXISTS messages
+                self.server_sent_unsolicited_exists(context);
+
+                loop {
+                    self.fetch_new_messages(context, folder.name(), false)
+                        .await
+                        .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
+
+                    // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
+                    if !self.server_sent_unsolicited_exists(context) {
+                        break;
+                    }
                 }
             }
         }
 
-        context
-            .set_config(Config::ConfiguredSentboxFolder, sentbox_folder.as_deref())
-            .await?;
-        context
-            .set_config(Config::ConfiguredSpamFolder, spam_folder.as_deref())
-            .await?;
+        // We iterate over both folder meanings to make sure that if e.g. the "Sent" folder was deleted,
+        // `ConfiguredSentboxFolder` is set to `None`:
+        for config in &[
+            Config::ConfiguredSentboxFolder,
+            Config::ConfiguredSpamFolder,
+        ] {
+            context
+                .set_config(*config, folder_configs.get(config).map(|s| s.as_str()))
+                .await?;
+        }
 
         last_scan.replace(Instant::now());
         Ok(())
     }
 }
 
-async fn get_watched_folders(context: &Context) -> Vec<String> {
+pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
     let mut res = Vec::new();
     let folder_watched_configured = &[
         (Config::SentboxWatch, Config::ConfiguredSentboxFolder),

src/imap/select_folder.rs

@@ -26,7 +26,7 @@ impl Imap {
     /// Issues a CLOSE command to expunge selected folder.
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
-    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
+    /// <https://tools.ietf.org/html/rfc3501#section-6.4.2>
     pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
@@ -37,7 +37,7 @@ impl Imap {
                         info!(context, "close/expunge succeeded");
                     }
                     Err(err) => {
-                        self.trigger_reconnect();
+                        self.trigger_reconnect(context).await;
                         return Err(Error::CloseExpungeFailed(err));
                     }
                 }
@@ -62,23 +62,23 @@ impl Imap {
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     /// Returns whether a new folder was selected.
-    pub(super) async fn select_folder<S: AsRef<str>>(
+    pub(super) async fn select_folder(
         &mut self,
         context: &Context,
-        folder: Option<S>,
+        folder: Option<&str>,
     ) -> Result<NewlySelected> {
         if self.session.is_none() {
             self.config.selected_folder = None;
             self.config.selected_folder_needs_expunge = false;
-            self.trigger_reconnect();
+            self.trigger_reconnect(context).await;
             return Err(Error::NoSession);
         }
 
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
-        if let Some(ref folder) = folder {
+        if let Some(folder) = folder {
             if let Some(ref selected_folder) = self.config.selected_folder {
-                if folder.as_ref() == selected_folder {
+                if folder == selected_folder {
                     return Ok(NewlySelected::No);
                 }
             }
@@ -88,31 +88,31 @@ impl Imap {
         self.maybe_close_folder(context).await?;
 
         // select new folder
-        if let Some(ref folder) = folder {
+        if let Some(folder) = folder {
             if let Some(ref mut session) = &mut self.session {
                 let res = session.select(folder).await;
 
-                // https://tools.ietf.org/html/rfc3501#section-6.3.1
+                // <https://tools.ietf.org/html/rfc3501#section-6.3.1>
                 // says that if the server reports select failure we are in
                 // authenticated (not-select) state.
 
                 match res {
                     Ok(mailbox) => {
-                        self.config.selected_folder = Some(folder.as_ref().to_string());
+                        self.config.selected_folder = Some(folder.to_string());
                         self.config.selected_mailbox = Some(mailbox);
                         Ok(NewlySelected::Yes)
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
-                        self.trigger_reconnect();
+                        self.trigger_reconnect(context).await;
                         self.config.selected_folder = None;
                         Err(Error::ConnectionLost)
                     }
                     Err(async_imap::error::Error::Validate(_)) => {
-                        Err(Error::BadFolderName(folder.as_ref().to_string()))
+                        Err(Error::BadFolderName(folder.to_string()))
                     }
                     Err(err) => {
                         self.config.selected_folder = None;
-                        self.trigger_reconnect();
+                        self.trigger_reconnect(context).await;
                         Err(Error::Other(err.to_string()))
                     }
                 }

src/imap/session.rs

@@ -3,6 +3,7 @@ use std::ops::{Deref, DerefMut};
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use async_std::net::TcpStream;
+use fast_socks5::client::Socks5Stream;
 
 #[derive(Debug)]
 pub(crate) struct Session {
@@ -17,6 +18,7 @@ pub(crate) trait SessionStream:
 impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
 impl SessionStream for TlsStream<TcpStream> {}
 impl SessionStream for TcpStream {}
+impl SessionStream for Socks5Stream<TcpStream> {}
 
 impl Deref for Session {
     type Target = ImapSession<Box<dyn SessionStream>>;

src/imex.rs

@@ -1,19 +1,20 @@
-//! # Import/export module
+//! # Import/export module.
 
 use std::any::Any;
 use std::ffi::OsStr;
 
+use ::pgp::types::KeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
-use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
+    path::{Path, PathBuf},
     prelude::*,
 };
+use async_tar::Archive;
 use rand::{thread_rng, Rng};
-use sqlx::Row;
 
-use crate::chat;
-use crate::chat::delete_and_reset_all_device_msgs;
+use crate::blob::BlobObject;
+use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
@@ -24,15 +25,13 @@ use crate::dc_tools::{
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
+use crate::log::LogExt;
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock_str;
-use crate::{blob::BlobObject, log::LogExt};
-use ::pgp::types::KeyTrait;
-use async_tar::Archive;
 
 // Name of the database file in the backup.
 const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
@@ -67,19 +66,19 @@ pub enum ImexMode {
 
 /// Import/export things.
 ///
-/// What to do is defined by the *what* parameter.
+/// What to do is defined by the `what` parameter.
 ///
 /// During execution of the job,
 /// some events are sent out:
 ///
-/// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
+/// - A number of `DC_EVENT_IMEX_PROGRESS` events are sent and may be used to create
 ///   a progress bar or stuff like that. Moreover, you'll be informed when the imex-job is done.
 ///
-/// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
+/// - For each file written on export, the function sends `DC_EVENT_IMEX_FILE_WRITTEN`
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(context: &Context, what: ImexMode, param1: impl AsRef<Path>) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: &Path) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
@@ -124,8 +123,7 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
-    let dir_name = dir_name.as_ref();
+pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
@@ -155,8 +153,7 @@ pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
-    let dir_name = dir_name.as_ref();
+pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
     let mut newest_backup_name = "".to_string();
@@ -187,7 +184,7 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
                             "Found backup file {} which could not be opened: {}", name, e
                         );
                         // On some Android devices we can't open sql files that are not in our private directory
-                        // (see https://github.com/deltachat/deltachat-android/issues/1768). So, compare names
+                        // (see <https://github.com/deltachat/deltachat-android/issues/1768>). So, compare names
                         // to still find the newest backup.
                         let name: String = name.into();
                         if newest_backup_time == 0
@@ -207,6 +204,7 @@ pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Re
     }
 }
 
+/// Initiates key transfer via Autocrypt Setup Message.
 pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
     use futures::future::FutureExt;
 
@@ -232,7 +230,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF).await?;
+    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
@@ -438,7 +436,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     Ok(plain_text)
 }
 
-pub fn normalize_setup_code(s: &str) -> String {
+fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
         if ('0'..='9').contains(&c) {
@@ -451,8 +449,8 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -> Result<()> {
-    info!(context, "Import/export dir: {}", path.as_ref().display());
+async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()> {
+    info!(context, "Import/export dir: {}", path.display());
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
@@ -476,12 +474,8 @@ async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -
 }
 
 /// Import Backup
-async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
-    if backup_to_import
-        .as_ref()
-        .to_string_lossy()
-        .ends_with(".bak")
-    {
+async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> {
+    if backup_to_import.to_string_lossy().ends_with(".bak") {
         // Backwards compability
         return import_backup_old(context, backup_to_import).await;
     }
@@ -489,7 +483,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     info!(
         context,
         "Import \"{}\" to \"{}\".",
-        backup_to_import.as_ref().display(),
+        backup_to_import.display(),
         context.get_dbfile().display()
     );
 
@@ -513,14 +507,16 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
+    let mut last_progress = 0;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
 
         let current_pos = f.raw_file_position();
         let progress = 1000 * current_pos / file_size;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
             context.emit_event(EventType::ImexProgress(progress as usize));
+            last_progress = progress;
         }
 
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
@@ -547,7 +543,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 
     context
         .sql
-        .open(context, &context.get_dbfile(), false)
+        .open(context, context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
@@ -556,11 +552,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     Ok(())
 }
 
-async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
-        backup_to_import.as_ref().display(),
+        backup_to_import.display(),
         context.get_dbfile().display()
     );
 
@@ -580,14 +576,14 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     );
 
     ensure!(
-        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()).await,
+        dc_copy_file(context, backup_to_import, context.get_dbfile()).await,
         "could not copy file"
     );
     /* error already logged */
     /* re-open copied database file */
     context
         .sql
-        .open(context, &context.get_dbfile(), false)
+        .open(context, context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
@@ -595,9 +591,8 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
 
     let total_files_cnt = context
         .sql
-        .count(sqlx::query("SELECT COUNT(*) FROM backup_blobs;"))
+        .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![])
         .await?;
-
     info!(
         context,
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
@@ -607,25 +602,33 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     // consuming too much memory.
     let file_ids = context
         .sql
-        .fetch(sqlx::query("SELECT id FROM backup_blobs ORDER BY id"))
-        .await?
-        .map(|row| row?.try_get(0))
-        .collect::<sqlx::Result<Vec<i64>>>()
+        .query_map(
+            "SELECT id FROM backup_blobs ORDER BY id",
+            paramsv![],
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
+                    .map_err(Into::into)
+            },
+        )
         .await?;
 
     let mut all_files_extracted = true;
     for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
         // Load a single blob into memory
-        let row = context
+        let (file_name, file_blob) = context
             .sql
-            .fetch_one(
-                sqlx::query("SELECT file_name, file_content FROM backup_blobs WHERE id = ?")
-                    .bind(file_id),
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| {
+                    let file_name: String = row.get(0)?;
+                    let file_blob: Vec<u8> = row.get(1)?;
+                    Ok((file_name, file_blob))
+                },
             )
             .await?;
 
-        let file_name: String = row.try_get(0)?;
-        let file_blob: &[u8] = row.try_get(1)?;
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -643,16 +646,16 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
         }
 
         let path_filename = context.get_blobdir().join(file_name);
-        dc_write_file(context, &path_filename, file_blob).await?;
+        dc_write_file(context, &path_filename, &file_blob).await?;
     }
 
     if all_files_extracted {
         // only delete backup_blobs if all files were successfully extracted
         context
             .sql
-            .execute(sqlx::query("DROP TABLE backup_blobs;"))
+            .execute("DROP TABLE backup_blobs;", paramsv![])
             .await?;
-        context.sql.execute(sqlx::query("VACUUM;")).await.ok();
+        context.sql.execute("VACUUM;", paramsv![]).await.ok();
         Ok(())
     } else {
         bail!("received stop signal");
@@ -663,7 +666,7 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
  * Export backup
  ******************************************************************************/
 #[allow(unused)]
-async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
     let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
@@ -677,7 +680,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
 
     context
         .sql
-        .execute(sqlx::query("VACUUM;"))
+        .execute("VACUUM;", paramsv![])
         .await
         .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
@@ -699,10 +702,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let res = export_backup_inner(context, &temp_path).await;
 
     // we re-open the database after export is finished
-    context
-        .sql
-        .open(context, &context.get_dbfile(), false)
-        .await;
+    context.sql.open(context, context.get_dbfile(), false).await;
 
     match &res {
         Ok(_) => {
@@ -739,6 +739,7 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
     let count = read_dir.len();
     let mut written_files = 0;
 
+    let mut last_progress = 0;
     for entry in read_dir.into_iter() {
         let entry = entry?;
         let name = entry.file_name();
@@ -756,9 +757,10 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 
         written_files += 1;
         let progress = 1000 * written_files / count;
-        if progress > 10 && progress < 1000 {
+        if progress != last_progress && progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
             emit_event!(context, EventType::ImexProgress(progress));
+            last_progress = progress;
         }
     }
 
@@ -769,7 +771,7 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
-async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
     /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
     plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
     Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
@@ -779,12 +781,12 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
     let mut set_default: bool;
     let mut imported_cnt = 0;
 
-    let dir_name = dir.as_ref().to_string_lossy();
+    let dir_name = dir.to_string_lossy();
     let mut dir_handle = async_std::fs::read_dir(&dir).await?;
     while let Some(entry) = dir_handle.next().await {
         let entry_fn = entry?.file_name();
         let name_f = entry_fn.to_string_lossy();
-        let path_plus_name = dir.as_ref().join(&entry_fn);
+        let path_plus_name = dir.join(&entry_fn);
         match dc_get_filesuffix_lc(&name_f) {
             Some(suffix) => {
                 if suffix != "asc" {
@@ -827,32 +829,35 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
     Ok(())
 }
 
-async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let mut export_errors = 0;
 
-    let mut keys = context
+    let keys = context
         .sql
-        .fetch(sqlx::query(
+        .query_map(
             "SELECT id, public_key, private_key, is_default FROM keypairs;",
-        ))
-        .await?
-        .map(|row| -> sqlx::Result<_> {
-            let row = row?;
-            let id = row.try_get(0)?;
-            let public_key_blob: &[u8] = row.try_get(1)?;
-            let public_key = SignedPublicKey::from_slice(public_key_blob);
-            let private_key_blob: &[u8] = row.try_get(2)?;
-            let private_key = SignedSecretKey::from_slice(private_key_blob);
-            let is_default: i32 = row.try_get(3)?;
+            paramsv![],
+            |row| {
+                let id = row.get(0)?;
+                let public_key_blob: Vec<u8> = row.get(1)?;
+                let public_key = SignedPublicKey::from_slice(&public_key_blob);
+                let private_key_blob: Vec<u8> = row.get(2)?;
+                let private_key = SignedSecretKey::from_slice(&private_key_blob);
+                let is_default: i32 = row.get(3)?;
 
-            Ok((id, public_key, private_key, is_default))
-        });
+                Ok((id, public_key, private_key, is_default))
+            },
+            |keys| {
+                keys.collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?;
 
-    while let Some(parts) = keys.next().await {
-        let (id, public_key, private_key, is_default) = parts?;
+    for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default != 0);
         if let Ok(key) = public_key {
-            if export_key_to_asc_file(context, &dir, id, &key)
+            if export_key_to_asc_file(context, dir, id, &key)
                 .await
                 .is_err()
             {
@@ -862,7 +867,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
             export_errors += 1;
         }
         if let Ok(key) = private_key {
-            if export_key_to_asc_file(context, &dir, id, &key)
+            if export_key_to_asc_file(context, dir, id, &key)
                 .await
                 .is_err()
             {
@@ -882,7 +887,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
  ******************************************************************************/
 async fn export_key_to_asc_file<T>(
     context: &Context,
-    dir: impl AsRef<Path>,
+    dir: &Path,
     id: Option<i64>,
     key: &T,
 ) -> std::io::Result<()>
@@ -899,7 +904,7 @@ where
             "unknown"
         };
         let id = id.map_or("default".into(), |i| i.to_string());
-        dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
+        dir.join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(
         context,
@@ -979,7 +984,7 @@ mod tests {
     async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
-        let blobdir = "$BLOBDIR";
+        let blobdir = Path::new("$BLOBDIR");
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
             .is_ok());
@@ -994,7 +999,7 @@ mod tests {
     async fn test_export_private_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().secret;
-        let blobdir = "$BLOBDIR";
+        let blobdir = Path::new("$BLOBDIR");
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
             .is_ok());
@@ -1009,7 +1014,7 @@ mod tests {
     async fn test_export_and_import_key() {
         let context = TestContext::new().await;
         context.configure_alice().await;
-        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let blobdir = context.ctx.get_blobdir();
         if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
             panic!("got error on export: {:?}", err);
         }

src/job.rs

@@ -1,4 +1,4 @@
-//! # Job module
+//! # Job module.
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
@@ -7,37 +7,36 @@ use std::{fmt, time::Duration};
 
 use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
 use async_smtp::smtp::response::{Category, Code, Detail};
-use async_std::prelude::*;
 use async_std::task::sleep;
+use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
-use sqlx::Row;
 
+use crate::blob::BlobObject;
+use crate::chat::{self, ChatId};
+use crate::config::Config;
+use crate::contact::{normalize_name, Contact, Modifier, Origin};
+use crate::context::Context;
 use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::ephemeral::load_imap_deletion_msgid;
 use crate::events::EventType;
 use crate::imap::{Imap, ImapActionResult};
 use crate::location;
-use crate::message::MsgId;
-use crate::message::{self, Message, MessageState};
+use crate::log::LogExt;
+use crate::message::{self, Message, MessageState, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
+use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
-use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
-use crate::{
-    chat::{self, Chat, ChatId, ChatItem},
-    constants::DC_CHAT_ID_DEADDROP,
-};
-use crate::{config::Config, constants::Blocked};
-use crate::{constants::Chattype, contact::Contact};
-use crate::{context::Context, log::LogExt};
-use crate::{scheduler::InterruptInfo, sql};
+use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
 
 /// Thread IDs
-#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[derive(
+    Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u32)]
 pub(crate) enum Thread {
     Unknown = 0,
@@ -75,7 +74,17 @@ impl Default for Thread {
 }
 
 #[derive(
-    Debug, Display, Copy, Clone, PartialEq, Eq, PartialOrd, FromPrimitive, ToPrimitive, sqlx::Type,
+    Debug,
+    Display,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
 )]
 #[repr(u32)]
 pub enum Action {
@@ -86,6 +95,9 @@ pub enum Action {
     FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
+    // this is user initiated so it should have a fairly high priority
+    UpdateRecentQuota = 140,
+
     // Moving message is prioritized lower than deletion so we don't
     // bother moving message if it is already scheduled for deletion.
     MoveMsg = 200,
@@ -121,6 +133,7 @@ impl From<Action> for Thread {
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
+            UpdateRecentQuota => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
@@ -139,7 +152,6 @@ pub struct Job {
     pub added_timestamp: i64,
     pub tries: u32,
     pub param: Params,
-    pub pending_error: Option<String>,
 }
 
 impl fmt::Display for Job {
@@ -160,7 +172,6 @@ impl Job {
             added_timestamp: timestamp,
             tries: 0,
             param,
-            pending_error: None,
         }
     }
 
@@ -173,7 +184,7 @@ impl Job {
         if self.job_id != 0 {
             context
                 .sql
-                .execute(sqlx::query("DELETE FROM jobs WHERE id=?;").bind(self.job_id as i32))
+                .execute("DELETE FROM jobs WHERE id=?;", paramsv![self.job_id as i32])
                 .await?;
         }
 
@@ -192,24 +203,26 @@ impl Job {
             context
                 .sql
                 .execute(
-                    sqlx::query(
-                        "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
-                    )
-                    .bind(self.desired_timestamp)
-                    .bind(self.tries as i64)
-                    .bind(self.param.to_string())
-                    .bind(self.job_id as i32),
+                    "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+                    paramsv![
+                        self.desired_timestamp,
+                        self.tries as i64,
+                        self.param.to_string(),
+                        self.job_id as i32,
+                    ],
                 )
                 .await?;
         } else {
             context.sql.execute(
-                sqlx::query("INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);")
-                    .bind(self.added_timestamp)
-                    .bind(thread)
-                    .bind(self.action)
-                    .bind(self.foreign_id)
-                    .bind(self.param.to_string())
-                    .bind(self.desired_timestamp)
+                "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
+                paramsv![
+                    self.added_timestamp,
+                    thread,
+                    self.action,
+                    self.foreign_id,
+                    self.param.to_string(),
+                    self.desired_timestamp
+                ]
             ).await?;
         }
 
@@ -237,11 +250,16 @@ impl Job {
             info!(context, "smtp-sending out mime message:");
             println!("{}", String::from_utf8_lossy(&message));
         }
-        let status = match smtp.send(context, recipients, message, job_id).await {
-            Err(crate::smtp::send::Error::SendError(err)) => {
+
+        smtp.connectivity.set_working(context).await;
+
+        let send_result = smtp.send(context, recipients, message, job_id).await;
+        smtp.last_send_error = send_result.as_ref().err().map(|e| e.to_string());
+
+        let status = match send_result {
+            Err(crate::smtp::send::Error::SmtpSend(err)) => {
                 // Remote error, retry later.
-                warn!(context, "SMTP failed to send: {:?}", err);
-                self.pending_error = Some(err.to_string());
+                warn!(context, "SMTP failed to send: {:?}", &err);
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
@@ -249,13 +267,13 @@ impl Job {
                         // instead of temporary ones.
                         let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
-                            // For documentation see https://tools.ietf.org/html/rfc3463
+                            // For documentation see <https://tools.ietf.org/html/rfc3463>
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
                             } => {
-                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Ignore status code 5.5.0, see <https://support.delta.chat/t/every-other-message-gets-stuck/877/2>
                                 // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
                                 // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
                                 //
@@ -289,7 +307,7 @@ impl Job {
                                 // Sometimes we receive transient errors that should be permanent.
                                 // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
                                 // receive as a transient error are misconfigurations of the smtp server.
-                                // See https://tools.ietf.org/html/rfc3463#section-3.2
+                                // See <https://tools.ietf.org/html/rfc3463#section-3.2>
                                 info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
                                 Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                             } else {
@@ -314,7 +332,7 @@ impl Job {
 
                 res
             }
-            Err(crate::smtp::send::Error::EnvelopeError(err)) => {
+            Err(crate::smtp::send::Error::Envelope(err)) => {
                 // Local error, job is invalid, do not retry.
                 smtp.disconnect().await;
                 warn!(context, "SMTP job is invalid: {}", err);
@@ -350,6 +368,7 @@ impl Job {
         //  SMTP server, if not yet done
         if let Err(err) = smtp.connect_configured(context).await {
             warn!(context, "SMTP connection failure: {:?}", err);
+            smtp.last_send_error = Some(format!("SMTP connection failure: {:#}", err));
             return Status::RetryLater;
         }
 
@@ -392,6 +411,8 @@ impl Job {
                 }
                 Err(err) => {
                     warn!(context, "failed to check message existence: {:?}", err);
+                    smtp.last_send_error =
+                        Some(format!("failed to check message existence: {:#}", err));
                     return Status::RetryLater;
                 }
             }
@@ -417,32 +438,39 @@ impl Job {
         &self,
         context: &Context,
         contact_id: u32,
-    ) -> sql::Result<(Vec<u32>, Vec<String>)> {
+    ) -> Result<(Vec<u32>, Vec<String>)> {
         // Extract message IDs from job parameters
-        let mut rows = context
+        let res: Vec<(u32, MsgId)> = context
             .sql
-            .fetch(
-                sqlx::query("SELECT id, param FROM jobs WHERE foreign_id=? AND id!=?")
-                    .bind(contact_id)
-                    .bind(self.job_id),
+            .query_map(
+                "SELECT id, param FROM jobs WHERE foreign_id=? AND id!=?",
+                paramsv![contact_id, self.job_id],
+                |row| {
+                    let job_id: u32 = row.get(0)?;
+                    let params_str: String = row.get(1)?;
+                    let params: Params = params_str.parse().unwrap_or_default();
+                    Ok((job_id, params))
+                },
+                |jobs| {
+                    let res = jobs
+                        .filter_map(|row| {
+                            let (job_id, params) = row.ok()?;
+                            let msg_id = params.get_msg_id()?;
+                            Some((job_id, msg_id))
+                        })
+                        .collect();
+                    Ok(res)
+                },
             )
             .await?;
 
         // Load corresponding RFC724 message IDs
         let mut job_ids = Vec::new();
         let mut rfc724_mids = Vec::new();
-
-        while let Some(row) = rows.next().await {
-            let row = row?;
-            let job_id: u32 = row.try_get(0)?;
-            let params_str: String = row.try_get(1)?;
-            let params: Params = params_str.parse().unwrap_or_default();
-            if let Some(msg_id) = params.get_msg_id() {
-                if let Ok(Message { rfc724_mid, .. }) = Message::load_from_db(context, msg_id).await
-                {
-                    job_ids.push(job_id);
-                    rfc724_mids.push(rfc724_mid);
-                }
+        for (job_id, msg_id) in res {
+            if let Ok(Message { rfc724_mid, .. }) = Message::load_from_db(context, msg_id).await {
+                job_ids.push(job_id);
+                rfc724_mids.push(rfc724_mid);
             }
         }
         Ok((job_ids, rfc724_mids))
@@ -499,6 +527,7 @@ impl Job {
         // connect to SMTP server, if not yet done
         if let Err(err) = smtp.connect_configured(context).await {
             warn!(context, "SMTP connection failure: {:?}", err);
+            smtp.last_send_error = Some(err.to_string());
             return Status::RetryLater;
         }
 
@@ -513,7 +542,7 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
+        if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
@@ -568,14 +597,15 @@ impl Job {
 
     /// Deletes a message on the server.
     ///
-    /// foreign_id is a MsgId pointing to a message in the trash chat
-    /// or a hidden message.
+    /// `foreign_id` is a MsgId.
     ///
-    /// This job removes the database record. If there are no more
-    /// records pointing to the same message on the server, the job
-    /// also removes the message on the server.
+    /// If the message is in the trash chat or hidden, this job
+    /// removes database record, otherwise it only clears the
+    /// `server_uid` column.  If there are no more records pointing to
+    /// the same message on the server, the job actually removes the
+    /// message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
+        if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
@@ -635,7 +665,6 @@ impl Job {
                 // Hidden messages are similar to trashed, but are
                 // related to some chat. We also delete their
                 // database records.
-                info!(context, "verbose (issue 2335): will delete from db");
                 job_try!(msg.id.delete_from_db(context).await)
             } else {
                 // Remove server UID from the database record.
@@ -646,7 +675,6 @@ impl Job {
                 // we remove UID to reduce the number of messages
                 // pointing to the corresponding UID. Once the counter
                 // reaches zero, we will remove the message.
-                info!(context, "verbose (issue 2335): will unlink");
                 job_try!(msg.id.unlink(context).await);
             }
             Status::Finished(Ok(()))
@@ -665,7 +693,7 @@ impl Job {
         if job_try!(context.get_config_bool(Config::Bot).await) {
             return Status::Finished(Ok(())); // Bots don't want those messages
         }
-        if let Err(err) = imap.connect_configured(context).await {
+        if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
@@ -690,38 +718,6 @@ impl Job {
             }
         }
 
-        // Make sure that if there now is a chat with a contact (created by an outgoing
-        // message), then group contact requests from this contact should also be unblocked.
-        // See https://github.com/deltachat/deltachat-core-rust/issues/2097.
-        for item in job_try!(chat::get_chat_msgs(context, DC_CHAT_ID_DEADDROP, 0, None).await) {
-            if let ChatItem::Message { msg_id } = item {
-                let msg = match Message::load_from_db(context, msg_id).await {
-                    Err(e) => {
-                        warn!(context, "can't get msg: {:#}", e);
-                        return Status::RetryLater;
-                    }
-                    Ok(m) => m,
-                };
-                let chat = match Chat::load_from_db(context, msg.chat_id).await {
-                    Err(e) => {
-                        warn!(context, "can't get chat: {:#}", e);
-                        return Status::RetryLater;
-                    }
-                    Ok(c) => c,
-                };
-                match chat.typ {
-                    Chattype::Group | Chattype::Mailinglist => {
-                        if let Ok((_1to1_chat, Blocked::Not)) =
-                            chat::lookup_by_contact_id(context, msg.from_id).await
-                        {
-                            chat.id.unblock(context).await;
-                        }
-                    }
-                    Chattype::Single | Chattype::Undefined => {}
-                }
-            }
-        }
-
         info!(context, "Done fetching existing messages.");
         Status::Finished(Ok(()))
     }
@@ -736,7 +732,7 @@ impl Job {
     /// Chat in contrast to the Sent folder, which is normally managed
     /// by the user via webmail or another email client.
     async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
+        if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
@@ -760,7 +756,7 @@ impl Job {
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
+        if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
@@ -822,29 +818,31 @@ impl Job {
 pub async fn kill_action(context: &Context, action: Action) -> bool {
     context
         .sql
-        .execute(sqlx::query("DELETE FROM jobs WHERE action=?;").bind(action))
+        .execute("DELETE FROM jobs WHERE action=?;", paramsv![action])
         .await
         .is_ok()
 }
 
 /// Remove jobs with specified IDs.
-async fn kill_ids(context: &Context, job_ids: &[u32]) -> sql::Result<()> {
+async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
         job_ids.iter().map(|_| "?").join(",")
     );
-    let mut query = sqlx::query(&q);
-    for id in job_ids {
-        query = query.bind(*id);
-    }
-    context.sql.execute(query).await?;
+    context
+        .sql
+        .execute(q, rusqlite::params_from_iter(job_ids))
+        .await?;
     Ok(())
 }
 
 pub async fn action_exists(context: &Context, action: Action) -> bool {
     context
         .sql
-        .exists(sqlx::query("SELECT COUNT(*) FROM jobs WHERE action=?;").bind(action))
+        .exists(
+            "SELECT COUNT(*) FROM jobs WHERE action=?;",
+            paramsv![action],
+        )
         .await
         .unwrap_or_default()
 }
@@ -853,7 +851,7 @@ async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
     message::update_msg_state(context, msg_id, MessageState::OutDelivered).await;
     let chat_id: ChatId = context
         .sql
-        .query_get_value(sqlx::query("SELECT chat_id FROM msgs WHERE id=?").bind(msg_id))
+        .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![msg_id])
         .await?
         .unwrap_or_default();
     context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
@@ -883,8 +881,8 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
 
                 match Contact::add_or_lookup(
                     context,
-                    display_name_normalized,
-                    contact.addr,
+                    &display_name_normalized,
+                    &contact.addr,
                     Origin::OutgoingTo,
                 )
                 .await
@@ -1030,18 +1028,10 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-pub(crate) async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
-    let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
-        info!(context, "verbose (issue 2335): loading imap deletion job");
-        Some(Job::new(
-            Action::DeleteMsgOnImap,
-            msg_id.to_u32(),
-            Params::new(),
-            0,
-        ))
-    } else {
-        None
-    };
+pub(crate) async fn load_imap_deletion_job(context: &Context) -> Result<Option<Job>> {
+    let res = load_imap_deletion_msgid(context)
+        .await?
+        .map(|msg_id| Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0));
     Ok(res)
 }
 
@@ -1142,7 +1132,7 @@ async fn perform_job_action(
 ) -> Status {
     info!(
         context,
-        "{} begin immediate try {} of job {:?} - verbose (issue 2335)", &connection, tries, job
+        "{} begin immediate try {} of job {}", &connection, tries, job
     );
 
     let try_res = match job.action {
@@ -1162,6 +1152,10 @@ async fn perform_job_action(
             sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
+        Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
+            Ok(status) => status,
+            Err(err) => Status::Finished(Err(err)),
+        },
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -1224,7 +1218,8 @@ pub async fn add(context: &Context, job: Job) {
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
-            | Action::MoveMsg => {
+            | Action::MoveMsg
+            | Action::UpdateRecentQuota => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))
@@ -1285,77 +1280,64 @@ pub(crate) async fn load_next(
         sleep(Duration::from_millis(500)).await;
     }
 
+    let query;
+    let params;
     let t = time();
+    let m;
     let thread_i = thread as i64;
 
-    let get_query = || {
-        if let Some(msg_id) = info.msg_id {
-            sqlx::query(
-                r#"
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND foreign_id=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-            .bind(msg_id)
-        } else if !info.probe_network {
-            // processing for first-try and after backoff-timeouts:
-            // process jobs in the order they were added.
-            sqlx::query(
-                r#"
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
+        // processing for first-try and after backoff-timeouts:
+        // process jobs in the order they were added.
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-            .bind(t)
-        } else {
-            // processing after call to dc_maybe_network():
-            // process _all_ pending jobs that failed before
-            // in the order of their backoff-times.
-            sqlx::query(
-                r#"
+"#;
+        params = paramsv![thread_i, t];
+    } else {
+        // processing after call to dc_maybe_network():
+        // process _all_ pending jobs that failed before
+        // in the order of their backoff-times.
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-        }
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
         let job_res = context
             .sql
-            .fetch_optional(get_query())
-            .await
-            .and_then(|row| {
-                if let Some(row) = row {
-                    Ok(Some(Job {
-                        job_id: row.try_get("id")?,
-                        action: row.try_get("action")?,
-                        foreign_id: row.try_get("foreign_id")?,
-                        desired_timestamp: row.try_get("desired_timestamp")?,
-                        added_timestamp: row.try_get("added_timestamp")?,
-                        tries: row.try_get::<i64, _>("tries")? as u32,
-                        param: row
-                            .try_get::<String, _>("param")?
-                            .parse()
-                            .unwrap_or_default(),
-                        pending_error: None,
-                    }))
-                } else {
-                    Ok(None)
-                }
-            });
+            .query_row_optional(query, params.clone(), |row| {
+                let job = Job {
+                    job_id: row.get("id")?,
+                    action: row.get("action")?,
+                    foreign_id: row.get("foreign_id")?,
+                    desired_timestamp: row.get("desired_timestamp")?,
+                    added_timestamp: row.get("added_timestamp")?,
+                    tries: row.get("tries")?,
+                    param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
+                };
+
+                Ok(job)
+            })
+            .await;
 
         match job_res {
             Ok(job) => break job,
@@ -1366,14 +1348,13 @@ LIMIT 1;
                 // TODO: improve by only doing a single query
                 match context
                     .sql
-                    .fetch_one(get_query())
+                    .query_row(query, params.clone(), |row| row.get::<_, i32>(0))
                     .await
-                    .and_then(|row| row.try_get::<i32, _>(0).map_err(Into::into))
                 {
                     Ok(id) => {
                         if let Err(err) = context
                             .sql
-                            .execute(sqlx::query("DELETE FROM jobs WHERE id=?;").bind(id))
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
                             .await
                         {
                             warn!(context, "failed to delete job {}: {:?}", id, err);
@@ -1401,14 +1382,9 @@ LIMIT 1;
                         .unwrap_or_default()
                         .or(Some(job))
                 } else {
-                    info!(context, "verbose (issue 2335): executing job normally");
                     Some(job)
                 }
             } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
-                info!(
-                    context,
-                    "verbose (issue 2335): loaded imap deletion job (no others queued)"
-                );
                 Some(job)
             } else {
                 load_housekeeping_job(context).await
@@ -1429,17 +1405,17 @@ mod tests {
         context
             .sql
             .execute(
-                sqlx::query(
-                    "INSERT INTO jobs
+                "INSERT INTO jobs
                    (added_timestamp, thread, action, foreign_id, param, desired_timestamp)
                  VALUES (?, ?, ?, ?, ?, ?);",
-                )
-                .bind(now)
-                .bind(Thread::from(Action::MoveMsg))
-                .bind(if valid { Action::MoveMsg as i32 } else { -1 })
-                .bind(foreign_id)
-                .bind(Params::new().to_string())
-                .bind(now),
+                paramsv![
+                    now,
+                    Thread::from(Action::MoveMsg),
+                    if valid { Action::MoveMsg as i32 } else { -1 },
+                    foreign_id,
+                    Params::new().to_string(),
+                    now
+                ],
             )
             .await
             .unwrap();
@@ -1459,7 +1435,7 @@ mod tests {
         )
         .await;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
-        assert!(jobs.unwrap().action == Action::Housekeeping);
+        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -1,55 +1,25 @@
-//! Cryptographic key module
+//! Cryptographic key module.
 
 use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
+use anyhow::{format_err, Result};
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
-use sqlx::Row;
-use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::KeyGenType;
 use crate::context::Context;
-use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
-use crate::sql;
+use crate::dc_tools::{time, EmailAddress};
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
-/// Error type for deltachat key handling.
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
-pub enum Error {
-    #[error("Could not decode base64")]
-    Base64Decode(#[from] base64::DecodeError),
-    #[error("rPGP error: {}", _0)]
-    Pgp(#[from] pgp::errors::Error),
-    #[error("Failed to generate PGP key: {}", _0)]
-    Keygen(#[from] crate::pgp::PgpKeygenError),
-    #[error("Failed to load key: {}", _0)]
-    LoadKey(#[from] sql::Error),
-    #[error("Failed to save generated key: {}", _0)]
-    StoreKey(#[from] SaveKeyError),
-    #[error("No address configured")]
-    NoConfiguredAddr,
-    #[error("Configured address is invalid: {}", _0)]
-    InvalidConfiguredAddr(#[from] InvalidEmailError),
-    #[error("no data provided")]
-    Empty,
-    #[error("db: {}", _0)]
-    Sql(#[from] sqlx::Error),
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
-}
-
-pub type Result<T> = std::result::Result<T, Error>;
-
 /// Convenience trait for working with keys.
 ///
 /// This trait is implemented for rPGP's [SignedPublicKey] and
@@ -80,7 +50,8 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+        Self::KeyType::from_armor_single(Cursor::new(bytes))
+            .map_err(|err| format_err!("rPGP error: {}", err))
     }
 
     /// Load the users' default key from the database.
@@ -123,17 +94,22 @@ impl DcKey for SignedPublicKey {
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
         match context
             .sql
-            .fetch_optional(sqlx::query(
+            .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            ))
+                paramsv![],
+                |row| {
+                    let bytes: Vec<u8> = row.get(0)?;
+                    Ok(bytes)
+                },
+            )
             .await?
         {
-            Some(row) => Self::from_slice(row.try_get(0)?),
+            Some(bytes) => Self::from_slice(&bytes),
             None => {
                 let keypair = generate_keypair(context).await?;
                 Ok(keypair.public)
@@ -165,17 +141,22 @@ impl DcKey for SignedSecretKey {
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
         match context
             .sql
-            .fetch_optional(sqlx::query(
+            .query_row_optional(
                 r#"
             SELECT private_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            ))
+                paramsv![],
+                |row| {
+                    let bytes: Vec<u8> = row.get(0)?;
+                    Ok(bytes)
+                },
+            )
             .await?
         {
-            Some(row) => Self::from_slice(row.try_get(0)?),
+            Some(bytes) => Self::from_slice(&bytes),
             None => {
                 let keypair = generate_keypair(context).await?;
                 Ok(keypair.secret)
@@ -221,30 +202,33 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await?
-        .ok_or(Error::NoConfiguredAddr)?;
+        .ok_or_else(|| format_err!("No address configured"))?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
     // Check if the key appeared while we were waiting on the lock.
     match context
         .sql
-        .fetch_optional(
-            sqlx::query(
-                r#"
+        .query_row_optional(
+            r#"
         SELECT public_key, private_key
           FROM keypairs
          WHERE addr=?1
            AND is_default=1;
         "#,
-            )
-            .bind(addr.to_string()),
+            paramsv![addr],
+            |row| {
+                let pub_bytes: Vec<u8> = row.get(0)?;
+                let sec_bytes: Vec<u8> = row.get(1)?;
+                Ok((pub_bytes, sec_bytes))
+            },
         )
         .await?
     {
-        Some(row) => Ok(KeyPair {
+        Some((pub_bytes, sec_bytes)) => Ok(KeyPair {
             addr,
-            public: SignedPublicKey::from_slice(row.try_get(0)?)?,
-            secret: SignedSecretKey::from_slice(row.try_get(1)?)?,
+            public: SignedPublicKey::from_slice(&pub_bytes)?,
+            secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         None => {
             let start = std::time::SystemTime::now();
@@ -277,24 +261,6 @@ pub enum KeyPairUse {
     ReadOnly,
 }
 
-/// Error saving a keypair to the database.
-#[derive(Debug, thiserror::Error)]
-#[error("SaveKeyError: {message}")]
-pub struct SaveKeyError {
-    message: String,
-    #[source]
-    cause: anyhow::Error,
-}
-
-impl SaveKeyError {
-    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
-        Self {
-            message: message.into(),
-            cause: cause.into(),
-        }
-    }
-}
-
 /// Store the keypair as an owned keypair for addr in the database.
 ///
 /// This will save the keypair as keys for the given address.  The
@@ -311,7 +277,7 @@ pub async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,
-) -> std::result::Result<(), SaveKeyError> {
+) -> Result<()> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
     let public_key = DcKey::to_bytes(&keypair.public);
@@ -319,18 +285,17 @@ pub async fn store_self_keypair(
     context
         .sql
         .execute(
-            sqlx::query("DELETE FROM keypairs WHERE public_key=? OR private_key=?;")
-                .bind(&public_key)
-                .bind(&secret_key),
+            "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
+            paramsv![public_key, secret_key],
         )
         .await
-        .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
+        .map_err(|err| err.context("failed to remove old use of key"))?;
     if default == KeyPairUse::Default {
         context
             .sql
-            .execute(sqlx::query("UPDATE keypairs SET is_default=0;"))
+            .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
-            .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
+            .map_err(|err| err.context("failed to clear default"))?;
     }
     let is_default = match default {
         KeyPairUse::Default => true as i32,
@@ -343,18 +308,12 @@ pub async fn store_self_keypair(
     context
         .sql
         .execute(
-            sqlx::query(
-                "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
+            "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
                 VALUES (?,?,?,?,?);",
-            )
-            .bind(addr)
-            .bind(is_default)
-            .bind(&public_key)
-            .bind(&secret_key)
-            .bind(t),
+            paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
-        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))?;
+        .map_err(|err| err.context("failed to insert keypair"))?;
 
     Ok(())
 }
@@ -364,10 +323,10 @@ pub async fn store_self_keypair(
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
-    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+    pub fn new(v: Vec<u8>) -> Result<Fingerprint> {
         match v.len() {
             20 => Ok(Fingerprint(v)),
-            _ => Err(FingerprintError::WrongLength),
+            _ => Err(format_err!("Wrong fingerprint length")),
         }
     }
 
@@ -406,7 +365,7 @@ impl fmt::Display for Fingerprint {
 
 /// Parse a human-readable or otherwise formatted fingerprint.
 impl std::str::FromStr for Fingerprint {
-    type Err = FingerprintError;
+    type Err = anyhow::Error;
 
     fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
         let hex_repr: String = input
@@ -420,21 +379,11 @@ impl std::str::FromStr for Fingerprint {
     }
 }
 
-#[derive(Debug, Error)]
-pub enum FingerprintError {
-    #[error("Invalid hex characters")]
-    NotHex(#[from] hex::FromHexError),
-    #[error("Incorrect fingerprint lengths")]
-    WrongLength,
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::{alice_keypair, TestContext};
 
-    use std::error::Error;
-
     use async_std::sync::Arc;
     use once_cell::sync::Lazy;
 
@@ -625,7 +574,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
         let nrows = || async {
             ctx.sql
-                .count(sqlx::query("SELECT COUNT(*) FROM keypairs;"))
+                .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
                 .await
                 .unwrap()
         };
@@ -676,13 +625,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
             .unwrap();
         assert_eq!(fp, res);
 
-        let err = "1".parse::<Fingerprint>().err().unwrap();
-        match err {
-            FingerprintError::NotHex(_) => (),
-            _ => panic!("Wrong error"),
-        }
-        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
-        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+        assert!("1".parse::<Fingerprint>().is_err());
     }
 
     #[test]

src/keyring.rs

@@ -3,7 +3,7 @@
 use anyhow::Result;
 
 use crate::context::Context;
-use crate::key::{self, DcKey};
+use crate::key::DcKey;
 
 /// An in-memory keyring.
 ///
@@ -27,14 +27,14 @@ where
     }
 
     /// Create a new keyring with the the user's secret key loaded.
-    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>> {
         let mut keyring: Keyring<T> = Keyring::new();
         keyring.load_self(context).await?;
         Ok(keyring)
     }
 
     /// Load the user's key into the keyring.
-    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+    pub async fn load_self(&mut self, context: &Context) -> Result<()> {
         self.add(T::load_self(context).await?);
         Ok(())
     }

src/lib.rs

@@ -1,22 +1,34 @@
+//! # Delta Chat Core Library.
+
+#![forbid(unsafe_code)]
 #![deny(
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
-    clippy::needless_borrow,
-    unsafe_code
+    clippy::needless_borrow
+)]
+#![allow(
+    clippy::match_bool,
+    clippy::eval_order_dependence,
+    clippy::bool_assert_comparison
 )]
-#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
 #[macro_use]
 extern crate smallvec;
+#[macro_use]
+extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
 
+pub trait ToSql: rusqlite::ToSql + Send + Sync {}
+
+impl<T: rusqlite::ToSql + Send + Sync> ToSql for T {}
+
 #[macro_use]
 pub mod log;
 #[macro_use]
@@ -49,7 +61,7 @@ mod imap;
 pub mod imex;
 mod scheduler;
 #[macro_use]
-pub mod job;
+mod job;
 mod format_flowed;
 pub mod key;
 mod keyring;
@@ -65,6 +77,7 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
+pub mod quota;
 pub mod securejoin;
 mod simplify;
 mod smtp;

src/location.rs

@@ -1,11 +1,9 @@
-//! Location handling
+//! Location handling.
 use std::convert::TryFrom;
 
 use anyhow::{ensure, Error};
-use async_std::prelude::*;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
-use sqlx::Row;
 
 use crate::chat::{self, ChatId};
 use crate::config::Config;
@@ -192,7 +190,7 @@ impl Kml {
     }
 }
 
-// location streaming
+/// Enables location streaming in chat identified by `chat_id` for `seconds` seconds.
 pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
     if !(seconds < 0 || chat_id.is_special()) {
@@ -201,15 +199,15 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
         if context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE chats    \
+                "UPDATE chats    \
              SET locations_send_begin=?,        \
              locations_send_until=?  \
              WHERE id=?",
-                )
-                .bind(if 0 != seconds { now } else { 0 })
-                .bind(if 0 != seconds { now + seconds } else { 0 })
-                .bind(chat_id),
+                paramsv![
+                    if 0 != seconds { now } else { 0 },
+                    if 0 != seconds { now + seconds } else { 0 },
+                    chat_id,
+                ],
             )
             .await
             .is_ok()
@@ -223,7 +221,7 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
                     .unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
                 let stock_str = stock_str::msg_location_disabled(context).await;
-                chat::add_info_msg(context, chat_id, stock_str).await;
+                chat::add_info_msg(context, chat_id, stock_str, now).await;
             }
             context.emit_event(EventType::ChatModified(chat_id));
             if 0 != seconds {
@@ -262,17 +260,16 @@ pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<Cha
         Some(chat_id) => context
             .sql
             .exists(
-                sqlx::query("SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;")
-                    .bind(chat_id)
-                    .bind(time()),
+                "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
             )
             .await
             .unwrap_or_default(),
         None => context
             .sql
             .exists(
-                sqlx::query("SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;")
-                    .bind(time()),
+                "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
             )
             .await
             .unwrap_or_default(),
@@ -285,29 +282,28 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
     }
     let mut continue_streaming = false;
 
-    if let Ok(mut chats) = context
+    if let Ok(chats) = context
         .sql
-        .fetch(sqlx::query("SELECT id FROM chats WHERE locations_send_until>?;").bind(time()))
+        .query_map(
+            "SELECT id FROM chats WHERE locations_send_until>?;",
+            paramsv![time()],
+            |row| row.get::<_, i32>(0),
+            |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
         .await
-        .map(|rows| rows.map(|row| row?.try_get::<i32, _>(0)))
     {
-        while let Some(chat_id) = chats.next().await {
-            let chat_id = match chat_id {
-                Ok(id) => id,
-                Err(_) => break,
-            };
+        for chat_id in chats {
             if let Err(err) = context.sql.execute(
-                sqlx::query(
                     "INSERT INTO locations  \
-                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);"
-                )
-                    .bind(latitude)
-                    .bind(longitude)
-                    .bind(accuracy)
-                    .bind(time())
-                    .bind(chat_id)
-                    .bind(DC_CONTACT_ID_SELF)
-
+                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
+                    paramsv![
+                        latitude,
+                        longitude,
+                        accuracy,
+                        time(),
+                        chat_id,
+                        DC_CONTACT_ID_SELF,
+                    ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
@@ -342,50 +338,54 @@ pub async fn get_range(
         Some(contact_id) => (0, contact_id),
         None => (1, 0), // this contact_id is unused
     };
-
     let list = context
         .sql
-        .fetch(
-            sqlx::query(
-                "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
+        .query_map(
+            "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
              COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
              FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
              AND (? OR l.from_id=?) \
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
-            )
-            .bind(disable_chat_id)
-            .bind(chat_id)
-            .bind(disable_contact_id)
-            .bind(contact_id as i64)
-            .bind(timestamp_from)
-            .bind(timestamp_to),
+            paramsv![
+                disable_chat_id,
+                chat_id,
+                disable_contact_id,
+                contact_id as i32,
+                timestamp_from,
+                timestamp_to,
+            ],
+            |row| {
+                let msg_id = row.get(6)?;
+                let txt: String = row.get(9)?;
+                let marker = if msg_id != 0 && is_marker(&txt) {
+                    Some(txt)
+                } else {
+                    None
+                };
+                let loc = Location {
+                    location_id: row.get(0)?,
+                    latitude: row.get(1)?,
+                    longitude: row.get(2)?,
+                    accuracy: row.get(3)?,
+                    timestamp: row.get(4)?,
+                    independent: row.get(5)?,
+                    msg_id,
+                    contact_id: row.get(7)?,
+                    chat_id: row.get(8)?,
+                    marker,
+                };
+                Ok(loc)
+            },
+            |locations| {
+                let mut ret = Vec::new();
+
+                for location in locations {
+                    ret.push(location?);
+                }
+                Ok(ret)
+            },
         )
-        .await?
-        .map(|row| {
-            let row = row?;
-            let msg_id = row.try_get(6)?;
-            let txt: String = row.try_get(9)?;
-            let marker = if msg_id != 0 && is_marker(&txt) {
-                Some(txt)
-            } else {
-                None
-            };
-            let loc = Location {
-                location_id: row.try_get(0)?,
-                latitude: row.try_get(1)?,
-                longitude: row.try_get(2)?,
-                accuracy: row.try_get(3)?,
-                timestamp: row.try_get(4)?,
-                independent: row.try_get(5)?,
-                msg_id,
-                contact_id: row.try_get(7)?,
-                chat_id: row.try_get(8)?,
-                marker,
-            };
-            Ok(loc)
-        })
-        .collect::<sqlx::Result<_>>()
         .await?;
     Ok(list)
 }
@@ -403,7 +403,7 @@ fn is_marker(txt: &str) -> bool {
 pub async fn delete_all(context: &Context) -> Result<(), Error> {
     context
         .sql
-        .execute(sqlx::query("DELETE FROM locations;"))
+        .execute("DELETE FROM locations;", paramsv![])
         .await?;
     context.emit_event(EventType::LocationChanged(None));
     Ok(())
@@ -417,65 +417,70 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
         .await?
         .unwrap_or_default();
 
-    let (locations_send_begin, locations_send_until, locations_last_sent) = {
-        let row = context.sql.fetch_one(
-            sqlx::query(
-                "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;"
-            )
-                .bind(chat_id)
-        ).await?;
+    let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
+        "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
+        paramsv![chat_id], |row| {
+            let send_begin: i64 = row.get(0)?;
+            let send_until: i64 = row.get(1)?;
+            let last_sent: i64 = row.get(2)?;
 
-        let send_begin: i64 = row.try_get(0)?;
-        let send_until: i64 = row.try_get(1)?;
-        let last_sent: i64 = row.try_get(2)?;
-
-        (send_begin, send_until, last_sent)
-    };
+            Ok((send_begin, send_until, last_sent))
+        })
+        .await?;
 
     let now = time();
     let mut location_count = 0;
     let mut ret = String::new();
     if locations_send_begin != 0 && now <= locations_send_until {
         ret += &format!(
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
+            <kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
             self_addr,
         );
 
-        let mut rows = context.sql.fetch(
-            sqlx::query(
+        context
+            .sql
+            .query_map(
                 "SELECT id, latitude, longitude, accuracy, timestamp \
              FROM locations  WHERE from_id=? \
              AND timestamp>=? \
-             AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
+             AND (timestamp>=? OR \
+                  timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
              AND independent=0 \
              GROUP BY timestamp \
-             ORDER BY timestamp;"
+             ORDER BY timestamp;",
+                paramsv![
+                    DC_CONTACT_ID_SELF,
+                    locations_send_begin,
+                    locations_last_sent,
+                    DC_CONTACT_ID_SELF
+                ],
+                |row| {
+                    let location_id: i32 = row.get(0)?;
+                    let latitude: f64 = row.get(1)?;
+                    let longitude: f64 = row.get(2)?;
+                    let accuracy: f64 = row.get(3)?;
+                    let timestamp = get_kml_timestamp(row.get(4)?);
+
+                    Ok((location_id, latitude, longitude, accuracy, timestamp))
+                },
+                |rows| {
+                    for row in rows {
+                        let (location_id, latitude, longitude, accuracy, timestamp) = row?;
+                        ret += &format!(
+                            "<Placemark>\
+                <Timestamp><when>{}</when></Timestamp>\
+                <Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point>\
+                </Placemark>\n",
+                            timestamp, accuracy, longitude, latitude
+                        );
+                        location_count += 1;
+                        last_added_location_id = location_id as u32;
+                    }
+                    Ok(())
+                },
             )
-                .bind(DC_CONTACT_ID_SELF)
-                .bind(locations_send_begin)
-                .bind(locations_last_sent)
-                .bind(DC_CONTACT_ID_SELF)
-        ).await?;
-
-        while let Some(row) = rows.next().await {
-            let row = row?;
-            let location_id: u32 = row.try_get(0)?;
-            let latitude: f64 = row.try_get(1)?;
-            let longitude: f64 = row.try_get(2)?;
-            let accuracy: f64 = row.try_get(3)?;
-            let timestamp = get_kml_timestamp(row.try_get(4)?);
-
-            ret += &format!(
-                "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
-                timestamp,
-                accuracy,
-                longitude,
-                latitude
-            );
-            location_count += 1;
-            last_added_location_id = location_id;
-        }
-
+            .await?;
         ret += "</Document>\n</kml>";
     }
 
@@ -516,9 +521,8 @@ pub async fn set_kml_sent_timestamp(
     context
         .sql
         .execute(
-            sqlx::query("UPDATE chats SET locations_last_sent=? WHERE id=?;")
-                .bind(timestamp)
-                .bind(chat_id),
+            "UPDATE chats SET locations_last_sent=? WHERE id=?;",
+            paramsv![timestamp, chat_id],
         )
         .await?;
     Ok(())
@@ -532,9 +536,8 @@ pub async fn set_msg_location_id(
     context
         .sql
         .execute(
-            sqlx::query("UPDATE msgs SET location_id=? WHERE id=?;")
-                .bind(location_id)
-                .bind(msg_id),
+            "UPDATE msgs SET location_id=? WHERE id=?;",
+            paramsv![location_id, msg_id],
         )
         .await?;
 
@@ -553,7 +556,6 @@ pub async fn save(
     let mut newest_timestamp = 0;
     let mut newest_location_id = 0;
 
-    let stmt_test = "SELECT COUNT(*) FROM locations WHERE timestamp=? AND from_id=?";
     let stmt_insert = "INSERT INTO locations\
              (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
              VALUES (?,?,?,?,?,?,?);";
@@ -566,28 +568,31 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        let exists = context
-            .sql
-            .exists(sqlx::query(stmt_test).bind(timestamp).bind(contact_id))
-            .await?;
+
+        let conn = context.sql.get_conn().await?;
+        let mut stmt_test =
+            conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
+        let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
+
+        let exists = stmt_test.exists(paramsv![timestamp, contact_id as i32])?;
+
         if independent || !exists {
-            let row_id = context
-                .sql
-                .insert(
-                    sqlx::query(stmt_insert)
-                        .bind(timestamp)
-                        .bind(contact_id)
-                        .bind(chat_id)
-                        .bind(latitude)
-                        .bind(longitude)
-                        .bind(accuracy)
-                        .bind(independent),
-                )
-                .await?;
+            stmt_insert.execute(paramsv![
+                timestamp,
+                contact_id as i32,
+                chat_id,
+                latitude,
+                longitude,
+                accuracy,
+                independent,
+            ])?;
 
             if timestamp > newest_timestamp {
+                // okay to drop, as we use cached prepared statements
+                drop(stmt_test);
+                drop(stmt_insert);
                 newest_timestamp = timestamp;
-                newest_location_id = row_id;
+                newest_location_id = conn.last_insert_rowid();
             }
         }
     }
@@ -605,21 +610,15 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
 
     let rows = context
         .sql
-        .fetch(
-            sqlx::query(
-                "SELECT id, locations_send_begin, locations_last_sent \
+        .query_map(
+            "SELECT id, locations_send_begin, locations_last_sent \
          FROM chats \
          WHERE locations_send_until>?;",
-            )
-            .bind(now),
-        )
-        .await
-        .map(|rows| {
-            rows.map(|row| -> sqlx::Result<Option<_>> {
-                let row = row?;
-                let chat_id: ChatId = row.try_get(0)?;
-                let locations_send_begin: i64 = row.try_get(1)?;
-                let locations_last_sent: i64 = row.try_get(2)?;
+            paramsv![now],
+            |row| {
+                let chat_id: ChatId = row.get(0)?;
+                let locations_send_begin: i64 = row.get(1)?;
+                let locations_last_sent: i64 = row.get(2)?;
                 continue_streaming = true;
 
                 // be a bit tolerant as the timer may not align exactly with time(NULL)
@@ -628,53 +627,57 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
                 } else {
                     Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
                 }
-            })
-            .filter_map(|v| v.transpose())
-        });
+            },
+            |rows| {
+                rows.filter_map(|v| v.transpose())
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await;
 
-    let stmt = "SELECT COUNT(*) \
-         FROM locations \
-         WHERE from_id=? \
-         AND timestamp>=? \
-         AND timestamp>? \
-         AND independent=0 \
-         ORDER BY timestamp;";
-
-    if let Ok(mut rows) = rows {
+    if let Ok(rows) = rows {
         let mut msgs = Vec::new();
-        while let Some(row) = rows.next().await {
-            let (chat_id, locations_send_begin, locations_last_sent) = match row {
-                Ok(row) => row,
-                Err(_) => break,
-            };
-            let exists = context
-                .sql
-                .exists(
-                    sqlx::query(stmt)
-                        .bind(DC_CONTACT_ID_SELF)
-                        .bind(locations_send_begin)
-                        .bind(locations_last_sent),
-                )
-                .await
-                .unwrap_or_default(); // TODO: better error handling
 
-            if !exists {
-                // if there is no new location, there's nothing to send.
-                // however, maybe we want to bypass this test eg. 15 minutes
-            } else {
-                // pending locations are attached automatically to every message,
-                // so also to this empty text message.
-                // DC_CMD_LOCATION is only needed to create a nicer subject.
-                //
-                // for optimisation and to avoid flooding the sending queue,
-                // we could sending these messages only if we're really online.
-                // the easiest way to determine this, is to check for an empty message queue.
-                // (might not be 100%, however, as positions are sent combined later
-                // and dc_set_location() is typically called periodically, this is ok)
-                let mut msg = Message::new(Viewtype::Text);
-                msg.hidden = true;
-                msg.param.set_cmd(SystemMessage::LocationOnly);
-                msgs.push((chat_id, msg));
+        {
+            let conn = job_try!(context.sql.get_conn().await);
+
+            let mut stmt_locations = job_try!(conn.prepare_cached(
+                "SELECT id \
+     FROM locations \
+     WHERE from_id=? \
+     AND timestamp>=? \
+     AND timestamp>? \
+     AND independent=0 \
+             ORDER BY timestamp;",
+            ));
+
+            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
+                if !stmt_locations
+                    .exists(paramsv![
+                        DC_CONTACT_ID_SELF,
+                        *locations_send_begin,
+                        *locations_last_sent,
+                    ])
+                    .unwrap_or_default()
+                {
+                    // if there is no new location, there's nothing to send.
+                    // however, maybe we want to bypass this test eg. 15 minutes
+                } else {
+                    // pending locations are attached automatically to every message,
+                    // so also to this empty text message.
+                    // DC_CMD_LOCATION is only needed to create a nicer subject.
+                    //
+                    // for optimisation and to avoid flooding the sending queue,
+                    // we could sending these messages only if we're really online.
+                    // the easiest way to determine this, is to check for an empty message queue.
+                    // (might not be 100%, however, as positions are sent combined later
+                    // and dc_set_location() is typically called periodically, this is ok)
+                    let mut msg = Message::new(Viewtype::Text);
+                    msg.hidden = true;
+                    msg.param.set_cmd(SystemMessage::LocationOnly);
+                    msgs.push((*chat_id, msg));
+                }
             }
         }
 
@@ -702,18 +705,19 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
     let chat_id = ChatId::new(job.foreign_id);
 
-    let (send_begin, send_until) = job_try!(context
-        .sql
-        .fetch_one(
-            sqlx::query(
+    let (send_begin, send_until) = job_try!(
+        context
+            .sql
+            .query_row(
                 "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
+                paramsv![chat_id],
+                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
             )
-            .bind(chat_id)
-        )
-        .await
-        .and_then(|row| { Ok((row.try_get::<i64, _>(0)?, row.try_get::<i64, _>(1)?)) }));
+            .await
+    );
 
-    if !(send_begin != 0 && time() <= send_until) {
+    let now = time();
+    if !(send_begin != 0 && now <= send_until) {
         // still streaming -
         // may happen as several calls to dc_send_locations_to_chat()
         // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
@@ -723,18 +727,16 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 context
                     .sql
                     .execute(
-                        sqlx::query(
-                            "UPDATE chats \
+                        "UPDATE chats \
                              SET locations_send_begin=0, locations_send_until=0 \
-                             WHERE id=?"
-                        )
-                        .bind(chat_id)
+ WHERE id=?",
+                        paramsv![chat_id],
                     )
                     .await
             );
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            chat::add_info_msg(context, chat_id, stock_str).await;
+            chat::add_info_msg(context, chat_id, stock_str, now).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }

src/log.rs

@@ -1,4 +1,5 @@
-//! # Logging
+//! # Logging.
+
 use crate::context::Context;
 
 #[macro_export]
@@ -42,17 +43,6 @@ macro_rules! error {
     }};
 }
 
-#[macro_export]
-macro_rules! error_network {
-    ($ctx:expr, $msg:expr) => {
-        error_network!($ctx, $msg,)
-    };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
-        let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
-    }};
-}
-
 #[macro_export]
 macro_rules! emit_event {
     ($ctx:expr, $event:expr) => {
@@ -76,7 +66,7 @@ where
     /// Once it is, you can add `#[track_caller]` to helper functions that use one of the log helpers here
     /// so that the location of the caller can be seen in the log. (this won't work with the macros,
     /// like warn!(), since the file!() and line!() macros don't work with track_caller)  
-    /// See https://github.com/rust-lang/rust/issues/78840 for progress on this.
+    /// See <https://github.com/rust-lang/rust/issues/78840> for progress on this.
     #[track_caller]
     fn log_err(self, context: &Context, msg: &str) -> Result<T, E> {
         self.log_err_inner(context, Some(msg))
@@ -126,7 +116,7 @@ where
     }
 }
 
-impl<T: Default, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
+impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
     #[track_caller]
     fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E> {
         if let Err(e) = &self {

src/login_param.rs

@@ -1,10 +1,18 @@
-//! # Login parameters
+//! # Login parameters.
 
 use std::borrow::Cow;
 use std::fmt;
+use std::time::Duration;
 
 use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
+use anyhow::Result;
+
+use async_std::io;
+use async_std::net::TcpStream;
+
+pub use async_smtp::ServerAddress;
+use fast_socks5::client::Socks5Stream;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
@@ -43,6 +51,81 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
+#[derive(Default, Debug, Clone, PartialEq)]
+pub struct Socks5Config {
+    pub host: String,
+    pub port: u16,
+    pub user_password: Option<(String, String)>,
+}
+
+impl Socks5Config {
+    /// Reads SOCKS5 configuration from the database.
+    pub async fn from_database(context: &Context) -> Result<Option<Self>> {
+        let sql = &context.sql;
+
+        let enabled = sql.get_raw_config_bool("socks5_enabled").await?;
+        if enabled {
+            let host = sql.get_raw_config("socks5_host").await?.unwrap_or_default();
+            let port: u16 = sql
+                .get_raw_config_int("socks5_port")
+                .await?
+                .unwrap_or_default() as u16;
+            let user = sql.get_raw_config("socks5_user").await?.unwrap_or_default();
+            let password = sql
+                .get_raw_config("socks5_password")
+                .await?
+                .unwrap_or_default();
+
+            let socks5_config = Self {
+                host,
+                port,
+                user_password: if !user.is_empty() {
+                    Some((user, password))
+                } else {
+                    None
+                },
+            };
+            Ok(Some(socks5_config))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub async fn connect(
+        &self,
+        target_addr: &ServerAddress,
+        timeout: Option<Duration>,
+    ) -> io::Result<Socks5Stream<TcpStream>> {
+        self.to_async_smtp_socks5_config()
+            .connect(target_addr, timeout)
+            .await
+    }
+
+    pub fn to_async_smtp_socks5_config(&self) -> async_smtp::smtp::Socks5Config {
+        async_smtp::smtp::Socks5Config {
+            host: self.host.clone(),
+            port: self.port,
+            user_password: self.user_password.clone(),
+        }
+    }
+}
+
+impl fmt::Display for Socks5Config {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "host:{},port:{},user_password:{}",
+            self.host,
+            self.port,
+            if let Some(user_password) = self.user_password.clone() {
+                format!("user: {}, password: ***", user_password.0)
+            } else {
+                "user: None".to_string()
+            }
+        )
+    }
+}
+
 #[derive(Default, Debug, Clone, PartialEq)]
 pub struct LoginParam {
     pub addr: String,
@@ -50,14 +133,12 @@ pub struct LoginParam {
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
     pub provider: Option<&'static Provider>,
+    pub socks5_config: Option<Socks5Config>,
 }
 
 impl LoginParam {
     /// Read the login parameters from the database.
-    pub async fn from_database(
-        context: &Context,
-        prefix: impl AsRef<str>,
-    ) -> crate::sql::Result<Self> {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -132,6 +213,8 @@ impl LoginParam {
             .await?
             .and_then(|provider_id| get_provider_by_id(&provider_id));
 
+        let socks5_config = Socks5Config::from_database(context).await?;
+
         Ok(LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -152,15 +235,12 @@ impl LoginParam {
             },
             provider,
             server_flags,
+            socks5_config,
         })
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_to_database(
-        &self,
-        context: &Context,
-        prefix: impl AsRef<str>,
-    ) -> crate::sql::Result<()> {
+    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -317,7 +397,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_save_load_login_param() -> anyhow::Result<()> {
+    async fn test_save_load_login_param() -> Result<()> {
         let t = TestContext::new().await;
 
         let param = LoginParam {
@@ -340,6 +420,8 @@ mod tests {
             },
             server_flags: 0,
             provider: get_provider_by_id("example.com"),
+            // socks5_config is not saved by `save_to_database`, using default value
+            socks5_config: None,
         };
 
         param.save_to_database(&t, "foobar_").await?;

src/lot.rs

@@ -1,3 +1,7 @@
+//! # Legacy generic return values for C API.
+
+use deltachat_derive::{FromSql, ToSql};
+
 use crate::key::Fingerprint;
 
 /// An object containing a set of values.
@@ -20,7 +24,9 @@ pub struct Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -64,8 +70,10 @@ impl Lot {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(u32)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum LotState {
     // Default
     Undefined = 0,
@@ -104,6 +112,16 @@ pub enum LotState {
     /// text1=error string
     QrError = 400,
 
+    QrWithdrawVerifyContact = 500,
+
+    /// text1=groupname
+    QrWithdrawVerifyGroup = 502,
+
+    QrReviveVerifyContact = 510,
+
+    /// text1=groupname
+    QrReviveVerifyGroup = 512,
+
     // Message States
     MsgInFresh = 10,
     MsgInNoticed = 13,

src/mimefactory.rs

@@ -1,10 +1,10 @@
+//! # MIME message production.
+
 use std::convert::TryInto;
 
 use anyhow::{bail, ensure, format_err, Result};
-use async_std::prelude::*;
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
-use sqlx::Row;
 
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat};
@@ -85,6 +85,33 @@ pub struct RenderedEmail {
     pub subject: String,
 }
 
+#[derive(Debug, Clone, Default)]
+struct MessageHeaders {
+    /// Opportunistically protected headers.
+    ///
+    /// These headers are placed into encrypted part *if* the message is encrypted. Place headers
+    /// which are not needed before decryption (e.g. Chat-Group-Name) or are not interesting if the
+    /// message cannot be decrypted (e.g. Chat-Disposition-Notification-To) here.
+    ///
+    /// If the message is not encrypted, these headers are placed into IMF header section, so make
+    /// sure that the message will be encrypted if you place any sensitive information here.
+    pub protected: Vec<Header>,
+
+    /// Headers that must go into IMF header section.
+    ///
+    /// These are standard headers such as Date, In-Reply-To, References, which cannot be placed
+    /// anywhere else according to the standard. Placing headers here also allows them to be fetched
+    /// individually over IMAP without downloading the message body. This is why Chat-Version is
+    /// placed here.
+    pub unprotected: Vec<Header>,
+
+    /// Headers that MUST NOT go into IMF header section.
+    ///
+    /// These are large headers which may hit the header section size limit on the server, such as
+    /// Chat-User-Avatar with a base64-encoded image inside.
+    pub hidden: Vec<Header>,
+}
+
 impl<'a> MimeFactory<'a> {
     pub async fn from_msg(
         context: &Context,
@@ -115,42 +142,51 @@ impl<'a> MimeFactory<'a> {
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
         } else {
-            let mut rows = context
+            context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.authname, c.addr  \
+                .query_map(
+                    "SELECT c.authname, c.addr  \
                  FROM chats_contacts cc  \
                  LEFT JOIN contacts c ON cc.contact_id=c.id  \
                  WHERE cc.chat_id=? AND cc.contact_id>9;",
-                    )
-                    .bind(msg.chat_id),
+                    paramsv![msg.chat_id],
+                    |row| {
+                        let authname: String = row.get(0)?;
+                        let addr: String = row.get(1)?;
+                        Ok((authname, addr))
+                    },
+                    |rows| {
+                        for row in rows {
+                            let (authname, addr) = row?;
+                            if !recipients_contain_addr(&recipients, &addr) {
+                                recipients.push((authname, addr));
+                            }
+                        }
+                        Ok(())
+                    },
                 )
                 .await?;
-            while let Some(row) = rows.next().await {
-                let row = row?;
-                let authname: String = row.try_get(0)?;
-                let addr: String = row.try_get(1)?;
-                if !recipients_contain_addr(&recipients, &addr) {
-                    recipients.push((authname, addr));
-                }
-            }
 
             if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await? {
                 req_mdn = true;
             }
         }
-        let row = context
+        let (in_reply_to, references) = context
             .sql
-            .fetch_one(
-                sqlx::query("SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?")
-                    .bind(msg.id),
+            .query_row(
+                "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
+                paramsv![msg.id],
+                |row| {
+                    let in_reply_to: String = row.get(0)?;
+                    let references: String = row.get(1)?;
+
+                    Ok((
+                        render_rfc724_mid_list(&in_reply_to),
+                        render_rfc724_mid_list(&references),
+                    ))
+                },
             )
             .await?;
-        let (in_reply_to, references) = (
-            render_rfc724_mid_list(row.try_get(0)?),
-            render_rfc724_mid_list(row.try_get(1)?),
-        );
 
         let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
@@ -402,14 +438,7 @@ impl<'a> MimeFactory<'a> {
     }
 
     pub async fn render(mut self, context: &Context) -> Result<RenderedEmail> {
-        // Headers that are encrypted
-        // - Chat-*, except Chat-Version
-        // - Secure-Join*
-        // - Subject
-        let mut protected_headers: Vec<Header> = Vec::new();
-
-        // All other headers
-        let mut unprotected_headers: Vec<Header> = Vec::new();
+        let mut headers: MessageHeaders = Default::default();
 
         let from = Address::new_mailbox_with_name(
             self.from_displayname.to_string(),
@@ -432,14 +461,20 @@ impl<'a> MimeFactory<'a> {
             to.push(from.clone());
         }
 
-        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+        headers
+            .unprotected
+            .push(Header::new("MIME-Version".into(), "1.0".into()));
 
         if !self.references.is_empty() {
-            unprotected_headers.push(Header::new("References".into(), self.references.clone()));
+            headers
+                .unprotected
+                .push(Header::new("References".into(), self.references.clone()));
         }
 
         if !self.in_reply_to.is_empty() {
-            unprotected_headers.push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
+            headers
+                .unprotected
+                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
         }
 
         let date = chrono::Utc
@@ -447,22 +482,29 @@ impl<'a> MimeFactory<'a> {
             .unwrap()
             .to_rfc2822();
 
-        unprotected_headers.push(Header::new("Date".into(), date));
+        headers.unprotected.push(Header::new("Date".into(), date));
 
-        unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
+        headers
+            .unprotected
+            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::Mdn { .. } = self.loaded {
-            unprotected_headers.push(Header::new(
+            headers.unprotected.push(Header::new(
                 "Auto-Submitted".to_string(),
                 "auto-replied".to_string(),
             ));
+        } else if context.get_config_bool(Config::Bot).await? {
+            headers.unprotected.push(Header::new(
+                "Auto-Submitted".to_string(),
+                "auto-generated".to_string(),
+            ));
         }
 
         if self.req_mdn {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
             // eg. are just freetext and/or do not follow any standard.
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Disposition-Notification-To".into(),
                 self.from_addr.clone(),
             ));
@@ -490,10 +532,14 @@ impl<'a> MimeFactory<'a> {
         if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
-            unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
+            headers
+                .unprotected
+                .push(Header::new("Autocrypt".into(), aheader));
         }
 
-        protected_headers.push(Header::new("Subject".into(), encoded_subject));
+        headers
+            .protected
+            .push(Header::new("Subject".into(), encoded_subject));
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
@@ -502,23 +548,28 @@ impl<'a> MimeFactory<'a> {
 
         let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(context).await?;
         if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Ephemeral-Timer".to_string(),
                 duration.to_string(),
             ));
         }
 
-        unprotected_headers.push(Header::new(
+        headers.unprotected.push(Header::new(
             "Message-ID".into(),
             render_rfc724_mid(&rfc724_mid),
         ));
 
-        unprotected_headers.push(Header::new_with_value("To".into(), to).unwrap());
-        unprotected_headers.push(Header::new_with_value("From".into(), vec![from]).unwrap());
+        headers
+            .unprotected
+            .push(Header::new_with_value("To".into(), to).unwrap());
+        headers
+            .unprotected
+            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
         if let Some(sender_displayname) = &self.sender_displayname {
             let sender =
                 Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
-            unprotected_headers
+            headers
+                .unprotected
                 .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
         }
 
@@ -526,13 +577,8 @@ impl<'a> MimeFactory<'a> {
 
         let (main_part, parts) = match self.loaded {
             Loaded::Message { .. } => {
-                self.render_message(
-                    context,
-                    &mut protected_headers,
-                    &mut unprotected_headers,
-                    &grpimage,
-                )
-                .await?
+                self.render_message(context, &mut headers, &grpimage)
+                    .await?
             }
             Loaded::Mdn { .. } => (self.render_mdn(context).await?, Vec::new()),
         };
@@ -555,12 +601,19 @@ impl<'a> MimeFactory<'a> {
             )
         };
 
-        // Store protected headers in the inner message.
-        let mut message = protected_headers
-            .into_iter()
-            .fold(message, |message, header| message.header(header));
-
         let outer_message = if is_encrypted {
+            // Store protected headers in the inner message.
+            let message = headers
+                .protected
+                .into_iter()
+                .fold(message, |message, header| message.header(header));
+
+            // Add hidden headers to encrypted payload.
+            let mut message = headers
+                .hidden
+                .into_iter()
+                .fold(message, |message, header| message.header(header));
+
             // Add gossip headers in chats with multiple recipients
             if peerstates.len() > 1 && self.should_do_gossip(context).await? {
                 for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) {
@@ -594,11 +647,6 @@ impl<'a> MimeFactory<'a> {
                 "multipart/encrypted; protocol=\"application/pgp-encrypted\"".to_string(),
             ));
 
-            // Store the unprotected headers on the outer message.
-            let outer_message = unprotected_headers
-                .into_iter()
-                .fold(outer_message, |message, header| message.header(header));
-
             if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
                 info!(context, "mimefactory: outgoing message mime:");
                 let raw_message = message.clone().build().as_string();
@@ -633,11 +681,33 @@ impl<'a> MimeFactory<'a> {
                 )
                 .header(("Subject".to_string(), "...".to_string()))
         } else {
-            unprotected_headers
+            let message = if headers.hidden.is_empty() {
+                message
+            } else {
+                // Store hidden headers in the inner unencrypted message.
+                let message = headers
+                    .hidden
+                    .into_iter()
+                    .fold(message, |message, header| message.header(header));
+
+                PartBuilder::new()
+                    .message_type(MimeMultipartType::Mixed)
+                    .child(message.build())
+            };
+
+            // Store protected headers in the outer message.
+            headers
+                .protected
                 .into_iter()
                 .fold(message, |message, header| message.header(header))
         };
 
+        // Store the unprotected headers on the outer message.
+        let outer_message = headers
+            .unprotected
+            .into_iter()
+            .fold(outer_message, |message, header| message.header(header));
+
         let MimeFactory {
             last_added_location_id,
             ..
@@ -698,8 +768,7 @@ impl<'a> MimeFactory<'a> {
     async fn render_message(
         &mut self,
         context: &Context,
-        protected_headers: &mut Vec<Header>,
-        unprotected_headers: &mut Vec<Header>,
+        headers: &mut MessageHeaders,
         grpimage: &Option<String>,
     ) -> Result<(PartBuilder, Vec<PartBuilder>)> {
         let chat = match &self.loaded {
@@ -711,20 +780,26 @@ impl<'a> MimeFactory<'a> {
         let mut meta_part = None;
 
         if chat.is_protected() {
-            protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
+            headers
+                .protected
+                .push(Header::new("Chat-Verified".to_string(), "1".to_string()));
         }
 
         if chat.typ == Chattype::Group {
-            protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
-            protected_headers.push(Header::new("Chat-Group-Name".into(), encoded));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-Name".into(), encoded));
 
             match command {
                 SystemMessage::MemberRemovedFromGroup => {
                     let email_to_remove = self.msg.param.get(Param::Arg).unwrap_or_default();
                     if !email_to_remove.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Member-Removed".into(),
                             email_to_remove.into(),
                         ));
@@ -733,7 +808,7 @@ impl<'a> MimeFactory<'a> {
                 SystemMessage::MemberAddedToGroup => {
                     let email_to_add = self.msg.param.get(Param::Arg).unwrap_or_default();
                     if !email_to_add.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Member-Added".into(),
                             email_to_add.into(),
                         ));
@@ -746,7 +821,7 @@ impl<'a> MimeFactory<'a> {
                             "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>",
                             "vg-member-added",
                         );
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Secure-Join".to_string(),
                             "vg-member-added".to_string(),
                         ));
@@ -754,18 +829,18 @@ impl<'a> MimeFactory<'a> {
                 }
                 SystemMessage::GroupNameChanged => {
                     let old_name = self.msg.param.get(Param::Arg).unwrap_or_default();
-                    protected_headers.push(Header::new(
+                    headers.protected.push(Header::new(
                         "Chat-Group-Name-Changed".into(),
                         maybe_encode_words(old_name),
                     ));
                 }
                 SystemMessage::GroupImageChanged => {
-                    protected_headers.push(Header::new(
+                    headers.protected.push(Header::new(
                         "Chat-Content".to_string(),
                         "group-avatar-changed".to_string(),
                     ));
                     if grpimage.is_none() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Avatar".to_string(),
                             "0".to_string(),
                         ));
@@ -777,13 +852,13 @@ impl<'a> MimeFactory<'a> {
 
         match command {
             SystemMessage::LocationStreamingEnabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".into(),
                     "location-streaming-enabled".into(),
                 ));
             }
             SystemMessage::EphemeralTimerChanged => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "ephemeral-timer-changed".to_string(),
                 ));
@@ -792,18 +867,19 @@ impl<'a> MimeFactory<'a> {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
-                // See https://tools.ietf.org/html/rfc3834
+                // See <https://tools.ietf.org/html/rfc3834>
                 //
                 // Adding this header without encryption leaks some
                 // information about the message contents, but it can
                 // already be easily guessed from message timing and size.
-                unprotected_headers.push(Header::new(
+                headers.unprotected.push(Header::new(
                     "Auto-Submitted".to_string(),
                     "auto-generated".to_string(),
                 ));
             }
             SystemMessage::AutocryptSetupMessage => {
-                unprotected_headers
+                headers
+                    .unprotected
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
 
                 placeholdertext = Some(stock_str::ac_setup_msg_body(context).await);
@@ -816,11 +892,13 @@ impl<'a> MimeFactory<'a> {
                         context,
                         "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>", step,
                     );
-                    protected_headers.push(Header::new("Secure-Join".into(), step.into()));
+                    headers
+                        .protected
+                        .push(Header::new("Secure-Join".into(), step.into()));
 
                     let param2 = msg.param.get(Param::Arg2).unwrap_or_default();
                     if !param2.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             if step == "vg-request-with-auth" || step == "vc-request-with-auth" {
                                 "Secure-Join-Auth".into()
                             } else {
@@ -832,24 +910,26 @@ impl<'a> MimeFactory<'a> {
 
                     let fingerprint = msg.param.get(Param::Arg3).unwrap_or_default();
                     if !fingerprint.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Secure-Join-Fingerprint".into(),
                             fingerprint.into(),
                         ));
                     }
                     if let Some(id) = msg.param.get(Param::Arg4) {
-                        protected_headers.push(Header::new("Secure-Join-Group".into(), id.into()));
+                        headers
+                            .protected
+                            .push(Header::new("Secure-Join-Group".into(), id.into()));
                     };
                 }
             }
             SystemMessage::ChatProtectionEnabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "protection-enabled".to_string(),
                 ));
             }
             SystemMessage::ChatProtectionDisabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "protection-disabled".to_string(),
                 ));
@@ -867,17 +947,21 @@ impl<'a> MimeFactory<'a> {
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
             meta_part = Some(mail);
-            protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
-            protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+            headers
+                .protected
+                .push(Header::new("Chat-Content".into(), "sticker".into()));
         } else if self.msg.viewtype == Viewtype::VideochatInvitation {
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Content".into(),
                 "videochat-invitation".into(),
             ));
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Webrtc-Room".into(),
                 self.msg
                     .param
@@ -892,12 +976,16 @@ impl<'a> MimeFactory<'a> {
             || self.msg.viewtype == Viewtype::Video
         {
             if self.msg.viewtype == Viewtype::Voice {
-                protected_headers.push(Header::new("Chat-Voice-Message".into(), "1".into()));
+                headers
+                    .protected
+                    .push(Header::new("Chat-Voice-Message".into(), "1".into()));
             }
             let duration_ms = self.msg.param.get_int(Param::Duration).unwrap_or_default();
             if duration_ms > 0 {
                 let dur = duration_ms.to_string();
-                protected_headers.push(Header::new("Chat-Duration".into(), dur));
+                headers
+                    .protected
+                    .push(Header::new("Chat-Duration".into(), dur));
             }
         }
 
@@ -1008,13 +1096,15 @@ impl<'a> MimeFactory<'a> {
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_selfavatar_file(context, &path) {
-                    Ok((part, filename)) => {
-                        parts.push(part);
-                        protected_headers.push(Header::new("Chat-User-Avatar".into(), filename))
-                    }
+                    Ok(avatar) => headers.hidden.push(Header::new(
+                        "Chat-User-Avatar".into(),
+                        format!("base64:{}", avatar),
+                    )),
                     Err(err) => warn!(context, "mimefactory: cannot attach selfavatar: {}", err),
                 },
-                None => protected_headers.push(Header::new("Chat-User-Avatar".into(), "0".into())),
+                None => headers
+                    .protected
+                    .push(Header::new("Chat-User-Avatar".into(), "0".into())),
             }
         }
 
@@ -1060,10 +1150,13 @@ impl<'a> MimeFactory<'a> {
             self.msg.get_summarytext(context, 32).await
         };
         let p2 = stock_str::read_rcpt_mail_body(context, p1).await;
-        let message_text = format!("{}\r\n", p2);
+        let message_text = format!("{}\r\n", format_flowed(&p2));
         message = message.child(
             PartBuilder::new()
-                .content_type(&mime::TEXT_PLAIN_UTF_8)
+                .header((
+                    "Content-Type".to_string(),
+                    "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+                ))
                 .body(message_text)
                 .build(),
         );
@@ -1194,29 +1287,11 @@ async fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String)> {
-    let blob = BlobObject::from_path(context, path)?;
-    let filename_to_send = match blob.suffix() {
-        Some(suffix) => format!("avatar.{}", suffix),
-        None => "avatar".to_string(),
-    };
-    let mimetype = match message::guess_msgtype_from_suffix(blob.as_rel_path()) {
-        Some(res) => res.1.parse()?,
-        None => mime::APPLICATION_OCTET_STREAM,
-    };
+fn build_selfavatar_file(context: &Context, path: &str) -> Result<String> {
+    let blob = BlobObject::from_path(context, path.as_ref())?;
     let body = std::fs::read(blob.to_abs_path())?;
     let encoded_body = wrapped_base64_encode(&body);
-
-    let part = PartBuilder::new()
-        .content_type(&mimetype)
-        .header((
-            "Content-Disposition",
-            format!("attachment; filename=\"{}\"", &filename_to_send),
-        ))
-        .header(("Content-Transfer-Encoding", "base64"))
-        .body(encoded_body);
-
-    Ok((part, filename_to_send))
+    Ok(encoded_body)
 }
 
 fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool {
@@ -1263,8 +1338,8 @@ fn encode_words(word: &str) -> String {
     encoded_words::encode(word, None, encoded_words::EncodingFlag::Shortest, None)
 }
 
-fn needs_encoding(to_check: impl AsRef<str>) -> bool {
-    !to_check.as_ref().chars().all(|c| {
+fn needs_encoding(to_check: &str) -> bool {
+    !to_check.chars().all(|c| {
         c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' || c == '%'
     })
 }
@@ -1280,14 +1355,16 @@ fn maybe_encode_words(words: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatId;
+    use async_std::prelude::*;
 
+    use crate::chat::ChatId;
+    use crate::chatlist::Chatlist;
     use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::mimeparser::MimeMessage;
-    use crate::test_utils::TestContext;
-    use crate::{chatlist::Chatlist, test_utils::get_chat_msg};
+    use crate::test_utils::{get_chat_msg, TestContext};
 
+    use async_std::fs::File;
     use pretty_assertions::assert_eq;
 
     #[test]
@@ -1325,7 +1402,7 @@ mod tests {
             Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
         );
 
-        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        // Addresses should not be unnecessarily be encoded, see <https://github.com/deltachat/deltachat-core-rust/issues/1575>:
         assert_eq!(s, "a space <x@y.org>");
     }
 
@@ -1620,7 +1697,7 @@ mod tests {
                 .unwrap()
                 .0;
 
-        let chat_id = chat::create_by_contact_id(&t, contact_id).await.unwrap();
+        let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
         new_msg.set_text(Some("Hi".to_string()));
@@ -1731,9 +1808,8 @@ mod tests {
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
-        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
-            .await
-            .unwrap();
+        let chat_id = chats.get_chat_id(0);
+        chat_id.accept(context).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
         new_msg.set_text(Some("Hi".to_string()));
@@ -1789,7 +1865,7 @@ mod tests {
 
     #[test]
     fn test_no_empty_lines_in_header() {
-        // See https://github.com/deltachat/deltachat-core-rust/issues/2118
+        // See <https://github.com/deltachat/deltachat-core-rust/issues/2118>
         let to_tuples = [
             ("Nnnn", "nnn@ttttttttt.de"),
             ("😀 ttttttt", "ttttttt@rrrrrr.net"),
@@ -1838,4 +1914,59 @@ mod tests {
 
         assert!(!headers.lines().any(|l| l.trim().is_empty()));
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_unencrypted() -> anyhow::Result<()> {
+        // create chat with bob, set selfavatar
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+        let file = t.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&file).await?.write_all(bytes).await?;
+        t.set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+
+        // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+        // make sure, `Subject:` stays in the outer header (imf header)
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("this is the text!".to_string()));
+
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(3, "\r\n\r\n");
+        let outer = payload.next().unwrap();
+        let inner = payload.next().unwrap();
+        let body = payload.next().unwrap();
+
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Subject:").count(), 1);
+        assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+        assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 1);
+        assert_eq!(inner.match_indices("Subject:").count(), 0);
+
+        assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+        // if another message is sent, that one must not contain the avatar
+        // and no artificial multipart/mixed nesting
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(2, "\r\n\r\n");
+        let outer = payload.next().unwrap();
+        let body = payload.next().unwrap();
+
+        assert_eq!(outer.match_indices("text/plain").count(), 1);
+        assert_eq!(outer.match_indices("Subject:").count(), 1);
+        assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 0);
+        assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+        assert_eq!(body.match_indices("this is the text!").count(), 1);
+        assert_eq!(body.match_indices("text/plain").count(), 0);
+        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(body.match_indices("Subject:").count(), 0);
+
+        Ok(())
+    }
 }

src/mimeparser.rs

@@ -1,17 +1,18 @@
+//! # MIME message parsing module.
+
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::pin::Pin;
 
 use anyhow::{bail, Result};
-use charset::Charset;
+use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 use once_cell::sync::Lazy;
-use percent_encoding::percent_decode_str;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSE};
+use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
 use crate::contact::addr_normalize;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_filemeta, dc_truncate};
@@ -102,7 +103,9 @@ pub(crate) enum MailinglistType {
     None,
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 #[repr(u32)]
 pub enum SystemMessage {
     Unknown = 0,
@@ -146,7 +149,7 @@ impl MimeMessage {
         let mut from = Default::default();
         let mut chat_disposition_notification_to = None;
 
-        // init known headers with what mailparse provided us
+        // Parse IMF headers.
         MimeMessage::merge_headers(
             context,
             &mut headers,
@@ -156,6 +159,21 @@ impl MimeMessage {
             &mail.headers,
         );
 
+        // Parse hidden headers.
+        let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+        if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" {
+            if let Some(part) = mail.subparts.first() {
+                for field in &part.headers {
+                    let key = field.get_key().to_lowercase();
+
+                    // For now only Chat-User-Avatar can be hidden.
+                    if !headers.contains_key(&key) && key == "chat-user-avatar" {
+                        headers.insert(key.to_string(), field.get_value());
+                    }
+                }
+            }
+        }
+
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
         headers.remove("chat-verified");
@@ -180,7 +198,7 @@ impl MimeMessage {
                         }
 
                         // Handle any gossip headers if the mail was encrypted.  See section
-                        // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
+                        // "3.6 Key Gossip" of <https://autocrypt.org/autocrypt-spec-1.1.0.pdf>
                         // but only if the mail was correctly signed:
                         if !signatures.is_empty() {
                             let gossip_headers =
@@ -202,7 +220,7 @@ impl MimeMessage {
                         let mut throwaway_from = from.clone();
 
                         // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
-                        // See https://github.com/deltachat/deltachat-core-rust/issues/1790.
+                        // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
                         headers.remove("subject");
 
                         MimeMessage::merge_headers(
@@ -260,7 +278,7 @@ impl MimeMessage {
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
-        parser.parse_headers(context);
+        parser.parse_headers(context).await?;
 
         if warn_empty_signature && parser.signatures.is_empty() {
             for part in parser.parts.iter_mut() {
@@ -277,7 +295,7 @@ impl MimeMessage {
 
     /// Parses system messages.
     fn parse_system_message_headers(&mut self, context: &Context) {
-        if self.get(HeaderDef::AutocryptSetupMessage).is_some() {
+        if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() {
             self.parts = self
                 .parts
                 .iter()
@@ -293,7 +311,7 @@ impl MimeMessage {
             } else {
                 warn!(context, "could not determine ASM mime-part");
             }
-        } else if let Some(value) = self.get(HeaderDef::ChatContent) {
+        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             } else if value == "ephemeral-timer-changed" {
@@ -307,20 +325,20 @@ impl MimeMessage {
     }
 
     /// Parses avatar action headers.
-    fn parse_avatar_headers(&mut self) {
-        if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
-            self.group_avatar = self.avatar_action_from_header(header_value);
+    async fn parse_avatar_headers(&mut self, context: &Context) {
+        if let Some(header_value) = self.get_header(HeaderDef::ChatGroupAvatar).cloned() {
+            self.group_avatar = self.avatar_action_from_header(context, header_value).await;
         }
 
-        if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
-            self.user_avatar = self.avatar_action_from_header(header_value);
+        if let Some(header_value) = self.get_header(HeaderDef::ChatUserAvatar).cloned() {
+            self.user_avatar = self.avatar_action_from_header(context, header_value).await;
         }
     }
 
     fn parse_videochat_headers(&mut self) {
-        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
+        if let Some(value) = self.get_header(HeaderDef::ChatContent).cloned() {
             if value == "videochat-invitation" {
-                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
+                let instance = self.get_header(HeaderDef::ChatWebrtcRoom).cloned();
                 if let Some(part) = self.parts.first_mut() {
                     part.typ = Viewtype::VideochatInvitation;
                     part.param
@@ -377,11 +395,12 @@ impl MimeMessage {
         }
 
         if let Some(mut part) = self.parts.pop() {
-            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
+            if part.typ == Viewtype::Audio && self.get_header(HeaderDef::ChatVoiceMessage).is_some()
+            {
                 part.typ = Viewtype::Voice;
             }
-            if part.typ == Viewtype::Image {
-                if let Some(value) = self.get(HeaderDef::ChatContent) {
+            if part.typ == Viewtype::Image || part.typ == Viewtype::Gif {
+                if let Some(value) = self.get_header(HeaderDef::ChatContent) {
                     if value == "sticker" {
                         part.typ = Viewtype::Sticker;
                     }
@@ -391,7 +410,7 @@ impl MimeMessage {
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
             {
-                if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
+                if let Some(field_0) = self.get_header(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
                         part.param.set_int(Param::Duration, duration_ms);
@@ -403,9 +422,9 @@ impl MimeMessage {
         }
     }
 
-    fn parse_headers(&mut self, context: &Context) {
+    async fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context);
-        self.parse_avatar_headers();
+        self.parse_avatar_headers(context).await;
         self.parse_videochat_headers();
         self.squash_attachment_parts();
 
@@ -448,15 +467,20 @@ impl MimeMessage {
         if !self.decrypting_failed && !self.parts.is_empty() {
             if let Some(ref dn_to) = self.chat_disposition_notification_to {
                 if let Some(from) = self.from.get(0) {
-                    if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
-                        if let Some(part) = self.parts.last_mut() {
-                            part.param.set_int(Param::WantsMdn, 1);
+                    // Check that the message is not outgoing.
+                    if !context.is_self_addr(&from.addr).await? {
+                        if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
+                            if let Some(part) = self.parts.last_mut() {
+                                part.param.set_int(Param::WantsMdn, 1);
+                            }
+                        } else {
+                            warn!(
+                                context,
+                                "{} requested a read receipt to {}, ignoring",
+                                from.addr,
+                                dn_to.addr
+                            );
                         }
-                    } else {
-                        warn!(
-                            context,
-                            "{} requested a read receipt to {}, ignoring", from.addr, dn_to.addr
-                        );
                     }
                 }
             }
@@ -480,12 +504,58 @@ impl MimeMessage {
 
             self.parts.push(part);
         }
+
+        if self.header.contains_key("auto-submitted") {
+            for part in &mut self.parts {
+                part.param.set(Param::Bot, "1");
+            }
+        }
+
+        Ok(())
     }
 
-    fn avatar_action_from_header(&mut self, header_value: String) -> Option<AvatarAction> {
+    async fn avatar_action_from_header(
+        &mut self,
+        context: &Context,
+        header_value: String,
+    ) -> Option<AvatarAction> {
         if header_value == "0" {
             Some(AvatarAction::Delete)
+        } else if let Some(avatar) = header_value
+            .split_ascii_whitespace()
+            .collect::<String>()
+            .strip_prefix("base64:")
+            .map(base64::decode)
+        {
+            // Avatar sent directly in the header as base64.
+            if let Ok(decoded_data) = avatar {
+                let extension = if let Ok(format) = image::guess_format(&decoded_data) {
+                    if let Some(ext) = format.extensions_str().first() {
+                        format!(".{}", ext)
+                    } else {
+                        String::new()
+                    }
+                } else {
+                    String::new()
+                };
+                match BlobObject::create(context, &format!("avatar{}", extension), &decoded_data)
+                    .await
+                {
+                    Ok(blob) => Some(AvatarAction::Change(blob.as_name().to_string())),
+                    Err(err) => {
+                        warn!(
+                            context,
+                            "Could not save decoded avatar to blob file: {}", err
+                        );
+                        None
+                    }
+                }
+            } else {
+                None
+            }
         } else {
+            // Avatar sent in attachment, as previous versions of Delta Chat did.
+
             let mut i = 0;
             while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
@@ -522,12 +592,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_subject(&self) -> Option<String> {
-        self.get(HeaderDef::Subject)
+        self.get_header(HeaderDef::Subject)
             .filter(|s| !s.is_empty())
             .map(|s| s.to_string())
     }
 
-    pub fn get(&self, headerdef: HeaderDef) -> Option<&String> {
+    pub fn get_header(&self, headerdef: HeaderDef) -> Option<&String> {
         self.header.get(headerdef.get_headername())
     }
 
@@ -675,7 +745,7 @@ impl MimeMessage {
                 The second body part contains the control information necessary to
                 verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
-                https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
+                <https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html>
                 for background information why we use encrypted+signed) */
                 if let Some(first) = mail.subparts.get(0) {
                     any_part_added = self
@@ -836,13 +906,14 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let simplified_txt =
-                            if simplified_txt.len() > DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() {
-                                self.is_mime_modified = true;
-                                dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
-                            } else {
-                                simplified_txt
-                            };
+                        let simplified_txt = if simplified_txt.chars().count()
+                            > DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len()
+                        {
+                            self.is_mime_modified = true;
+                            dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
+                        } else {
+                            simplified_txt
+                        };
 
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
@@ -949,12 +1020,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_mailinglist_type(&self) -> MailinglistType {
-        if self.get(HeaderDef::ListId).is_some() {
+        if self.get_header(HeaderDef::ListId).is_some() {
             return MailinglistType::ListIdBased;
-        } else if self.get(HeaderDef::Sender).is_some() {
+        } else if self.get_header(HeaderDef::Sender).is_some() {
             // the `Sender:`-header alone is no indicator for mailing list
             // as also used for bot-impersonation via `set_override_sender_name()`
-            if let Some(precedence) = self.get(HeaderDef::Precedence) {
+            if let Some(precedence) = self.get_header(HeaderDef::Precedence) {
                 if precedence == "list" || precedence == "bulk" {
                     return MailinglistType::SenderBased;
                 }
@@ -980,8 +1051,8 @@ impl MimeMessage {
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
-        self.get(HeaderDef::XMicrosoftOriginalMessageId)
-            .or_else(|| self.get(HeaderDef::MessageId))
+        self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
+            .or_else(|| self.get_header(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
@@ -1168,7 +1239,7 @@ impl MimeMessage {
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
     #[allow(clippy::indexing_slicing)]
     async fn heuristically_parse_ndn(&mut self, context: &Context) {
-        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
+        let maybe_ndn = if let Some(from) = self.get_header(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
             from.contains("mailer-daemon") || from.contains("mail-daemon")
         } else {
@@ -1243,13 +1314,14 @@ impl MimeMessage {
     /// database, returns None.
     pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
         let parent_timestamp = if let Some(field) = self
-            .get(HeaderDef::InReplyTo)
+            .get_header(HeaderDef::InReplyTo)
             .and_then(|msgid| parse_message_id(msgid).ok())
         {
             context
                 .sql
                 .query_get_value(
-                    sqlx::query("SELECT timestamp FROM msgs WHERE rfc724_mid=?").bind(field),
+                    "SELECT timestamp FROM msgs WHERE rfc724_mid=?",
+                    paramsv![field],
                 )
                 .await?
         } else {
@@ -1286,7 +1358,9 @@ async fn update_gossip_peerstates(
                     peerstate = Some(p);
                 }
                 if let Some(peerstate) = peerstate {
-                    peerstate.handle_fingerprint_change(context).await?;
+                    peerstate
+                        .handle_fingerprint_change(context, message_time)
+                        .await?;
                 }
 
                 gossipped_addr.insert(header.addr.clone());
@@ -1406,7 +1480,7 @@ fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> {
         mime::VIDEO => Viewtype::Video,
         mime::MULTIPART => Viewtype::Unknown,
         mime::MESSAGE => {
-            // Enacapsulated messages, see https://www.w3.org/Protocols/rfc1341/7_3_Message.html
+            // Enacapsulated messages, see <https://www.w3.org/Protocols/rfc1341/7_3_Message.html>
             // Also used as part "message/disposition-notification" of "multipart/report", which, however, will
             // be handled separatedly.
             // I've not seen any messages using this, so we do not attach these parts (maybe they're used to attach replies,
@@ -1447,55 +1521,13 @@ fn get_attachment_filename(
     // `Content-Disposition: ... filename=...`
     let mut desired_filename = ct.params.get("filename").map(|s| s.to_string());
 
-    // try to get file name from
-    // `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
-    // encoded as CHARSET'LANG'test%2E%70%64%66 (key ends with `*`)
-    // or as "encoded-words" (key does not end with `*`)
     if desired_filename.is_none() {
-        let mut apostrophe_encoded = false;
-        desired_filename = ct
-            .params
-            .iter()
-            .filter(|(key, _value)| key.starts_with("filename*"))
-            .fold(None, |acc, (key, value)| {
-                if key.ends_with('*') {
-                    apostrophe_encoded = true;
-                }
-                if let Some(acc) = acc {
-                    Some(acc + value)
-                } else {
-                    Some(value.to_string())
-                }
-            });
-        if apostrophe_encoded {
-            if let Some(name) = desired_filename {
-                let mut parts = name.splitn(3, '\'');
-                desired_filename =
-                    if let (Some(charset), Some(value)) = (parts.next(), parts.last()) {
-                        let decoded_bytes = percent_decode_str(value);
-                        if charset.to_lowercase() == "utf-8" {
-                            Some(decoded_bytes.decode_utf8_lossy().to_string())
-                        } else {
-                            // encoded_words crate say, latin-1 is not reported; moreover, latin1 is a good default
-                            if let Some(charset) = Charset::for_label(charset.as_bytes())
-                                .or_else(|| Charset::for_label(b"latin1"))
-                            {
-                                let decoded_bytes = decoded_bytes.collect::<Vec<u8>>();
-                                let (utf8_str, _, _) = charset.decode(&*decoded_bytes);
-                                Some(utf8_str.into())
-                            } else {
-                                warn!(context, "latin1 encoding does not exist");
-                                None
-                            }
-                        }
-                    } else {
-                        warn!(context, "apostrophed encoding invalid: {}", name);
-                        // be graceful and just use the original name.
-                        // some MUA, including Delta Chat up to core1.50,
-                        // use `filename*` mistakenly for simple encoded-words without following rfc2231
-                        Some(name)
-                    }
-            }
+        if let Some(name) = ct.params.get("filename*").map(|s| s.to_string()) {
+            // be graceful and just use the original name.
+            // some MUA, including Delta Chat up to core1.50,
+            // use `filename*` mistakenly for simple encoded-words without following rfc2231
+            warn!(context, "apostrophed encoding invalid: {}", name);
+            desired_filename = Some(name);
         }
     }
 
@@ -1528,7 +1560,7 @@ fn get_attachment_filename(
 /// Returned addresses are normalized and lowercased.
 pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc" || header_key == "bcc"
+        header_key == "to" || header_key == "cc"
     })
 }
 
@@ -1576,7 +1608,6 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::constants::DC_MAX_GET_TEXT_LEN;
     use crate::{
         chatlist::Chatlist,
         config::Config,
@@ -1857,13 +1888,6 @@ mod tests {
         assert_eq!(filename, Some("Maßnahmen Okt. 2020.html".to_string()))
     }
 
-    #[test]
-    fn test_charset_latin1() {
-        // make sure, latin1 exists under this name
-        // as we're using it as default in get_attachment_filename() for non-utf-8
-        assert!(Charset::for_label(b"latin1").is_some());
-    }
-
     #[test]
     fn test_mailparse_content_type() {
         let ctype =
@@ -1920,9 +1944,8 @@ mod tests {
             .ctx
             .sql
             .execute(
-                sqlx::query("INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)")
-                    .bind("Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org")
-                    .bind(timestamp),
+                "INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)",
+                paramsv!["Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org", timestamp],
             )
             .await
             .expect("Failed to write to the database");
@@ -1960,11 +1983,11 @@ mod tests {
             .unwrap();
 
         // non-overwritten headers do not bubble up
-        let of = mimeparser.get(HeaderDef::SecureJoinGroup).unwrap();
+        let of = mimeparser.get_header(HeaderDef::SecureJoinGroup).unwrap();
         assert_eq!(of, "no");
 
         // unknown headers do not bubble upwards
-        let of = mimeparser.get(HeaderDef::_TestHeader).unwrap();
+        let of = mimeparser.get_header(HeaderDef::_TestHeader).unwrap();
         assert_eq!(of, "Bar");
 
         // the following fields would bubble up
@@ -1973,13 +1996,15 @@ mod tests {
         // for Chat-Version, also the case-insensivity is tested.
         assert_eq!(mimeparser.get_subject(), Some("outer-subject".into()));
 
-        let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
+        let of = mimeparser.get_header(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "0.0");
         assert_eq!(mimeparser.parts.len(), 1);
 
         // make sure, headers that are only allowed in the encrypted part
         // cannot be set from the outer part
-        assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none());
+        assert!(mimeparser
+            .get_header(HeaderDef::SecureJoinFingerprint)
+            .is_none());
     }
 
     #[async_std::test]
@@ -2772,7 +2797,7 @@ On 2020-10-25, Bob wrote:
         .unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let msg_id = chats.get_msg_id(0).unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap().unwrap();
         let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
 
         assert_eq!(
@@ -2782,7 +2807,7 @@ On 2020-10-25, Bob wrote:
         assert_eq!(msg.viewtype, Viewtype::Image);
         assert_eq!(msg.error(), None);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
-        assert_eq!(msg.chat_blocked, Blocked::Deaddrop);
+        assert_eq!(msg.chat_blocked, Blocked::Request);
         assert_eq!(msg.state, MessageState::InFresh);
         assert_eq!(msg.get_filebytes(&t).await, 2115);
         assert!(msg.get_file(&t).is_some());
@@ -2860,21 +2885,20 @@ On 2020-10-25, Bob wrote:
         let t = TestContext::new().await;
 
         static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n";
-        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_MAX_GET_TEXT_LEN
+        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_DESIRED_TEXT_LEN
         let long_txt = format!("From: alice@c.de\n\n{}", REPEAT_TXT.repeat(REPEAT_CNT));
-        assert!(DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() < DC_MAX_GET_TEXT_LEN);
 
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref())
             .await
             .unwrap();
         assert_eq!(long_txt.matches("just repeated").count(), REPEAT_CNT);
-        assert!(long_txt.len() > DC_MAX_GET_TEXT_LEN);
+        assert!(long_txt.len() > DC_DESIRED_TEXT_LEN);
         assert!(mimemsg.is_mime_modified);
         assert!(
             mimemsg.parts[0].msg.matches("just repeated").count()
-                < DC_MAX_GET_TEXT_LEN / REPEAT_TXT.len()
+                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_MAX_GET_TEXT_LEN);
+        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
     }
 
     #[async_std::test]
@@ -2899,4 +2923,78 @@ On 2020-10-25, Bob wrote:
             Some("Mr.6Dx7ITn4w38.n9j7epIcuQI@outlook.com".to_string())
         );
     }
+
+    #[async_std::test]
+    async fn test_long_in_reply_to() -> Result<()> {
+        let t = TestContext::new_alice().await;
+
+        // A message with a long Message-ID.
+        // Long message-IDs are generated by Mailjet.
+        let raw = br###"Date: Thu, 28 Jan 2021 00:26:57 +0000
+Chat-Version: 1.0\n\
+Message-ID: <ABCDEFGH.1234567_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA@mailjet.com>
+To: Bob <bob@example.org>
+From: Alice <alice@example.org>
+Subject: ...
+
+Some quote.
+"###;
+        dc_receive_imf(&t, raw, "INBOX", 1, false).await?;
+
+        // Delta Chat generates In-Reply-To with a starting tab when Message-ID is too long.
+        let raw = br###"In-Reply-To:
+	<ABCDEFGH.1234567_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA@mailjet.com>
+Date: Thu, 28 Jan 2021 00:26:57 +0000
+Chat-Version: 1.0\n\
+Message-ID: <foobar@example.org>
+To: Alice <alice@example.org>
+From: Bob <bob@example.org>
+Subject: ...
+
+> Some quote.
+
+Some reply
+"###;
+
+        dc_receive_imf(&t, raw, "INBOX", 2, false).await?;
+
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.get_text().unwrap(), "Some reply");
+        let quoted_message = msg.quoted_message(&t).await?.unwrap();
+        assert_eq!(quoted_message.get_text().unwrap(), "Some quote.");
+
+        Ok(())
+    }
+
+    // Test that WantsMdn parameter is not set on outgoing messages.
+    #[async_std::test]
+    async fn test_outgoing_wants_mdn() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let raw = br###"Date: Thu, 28 Jan 2021 00:26:57 +0000
+Chat-Version: 1.0\n\
+Message-ID: <foobarbaz@example.org>
+To: Bob <bob@example.org>
+From: Alice <alice@example.com>
+Subject: subject
+Chat-Disposition-Notification-To: alice@example.com
+
+Message.
+"###;
+
+        // Bob receives message.
+        dc_receive_imf(&bob, raw, "INBOX", 1, false).await?;
+        let msg = bob.get_last_msg().await;
+        // Message is incoming.
+        assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
+
+        // Alice receives copy-to-self.
+        dc_receive_imf(&alice, raw, "INBOX", 1, false).await?;
+        let msg = alice.get_last_msg().await;
+        // Message is outgoing, don't send read receipt to self.
+        assert!(msg.param.get_bool(Param::WantsMdn).is_none());
+
+        Ok(())
+    }
 }