also export qoutes, message state and is_forwarded

step, this could decrease memory usage

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
     executor: doxygen
     steps:
       - checkout
-      - run: bash ci_scripts/run-doxygen.sh
+      - run: bash scripts/run-doxygen.sh
       - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
@@ -27,7 +27,10 @@ jobs:
       - checkout
       # the following commands on success produces
       # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
+      - run:
+          # building aarch64 packages under qemu is very slow
+          no_output_timeout: 60m
+          command: bash scripts/remote_python_packaging.sh
       - persist_to_workspace:
           root: workspace
           paths:
@@ -42,7 +45,7 @@ jobs:
       - attach_workspace:
           at: workspace
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
 workflows:
   version: 2.1
@@ -51,18 +54,24 @@ workflows:
     jobs:
       - remote_python_packaging:
           filters:
+            tags:
+              only: /.+/
             branches:
-              only: master
+              ignore: /.*/
+
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
 
       - upload_docs_wheels:
           requires:
              - remote_python_packaging
              - build_doxygen
           filters:
+            tags:
+              only: /.+/
             branches:
-              only: master
-
-      - build_doxygen:
-          filters:
-            branches:
-              only: master
+              ignore: /.*/

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "cargo"
+    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -65,26 +65,16 @@ jobs:
 
   build_and_test:
     name: Build and test
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        # macOS disabled due to random failures related to caching
-        #os: [ubuntu-latest, windows-latest, macOS-latest]
-        os: [ubuntu-latest, windows-latest]
-        rust: [1.50.0]
-        experimental: [false]
-#        include:
-#          - os: ubuntu-latest
-#            rust: nightly
-#            experimental: true
-#          - os: windows-latest
-#            rust: nightly
-#            experimental: true
-#          - os: macOS-latest
-#            rust: nightly
-#            experimental: true
-
+        include:
+          - os: ubuntu-latest
+            rust: 1.50.0
+            python: 3.6
+          - os: windows-latest
+            rust: 1.50.0
+            python: false # Python bindings compilation on Windows is not supported.
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -114,8 +104,10 @@ jobs:
 
     - name: check
       uses: actions-rs/cargo@v1
+      env:
+        RUSTFLAGS: -D warnings
       with:
-        command:  check
+        command: check
         args: --all --bins --examples --tests --features repl
 
     - name: tests
@@ -123,3 +115,29 @@ jobs:
       with:
         command: test
         args: --all
+
+    - name: install python
+      if: ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: install tox
+      if: ${{ matrix.python }}
+      run: pip install tox
+
+    - name: build C library
+      if: ${{ matrix.python }}
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: -p deltachat_ffi
+
+    - name: run python tests
+      if: ${{ matrix.python }}
+      env:
+        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        DCC_RS_TARGET: debug
+        DCC_RS_DEV: ${{ github.workspace }}
+      working-directory: python
+      run: tox -e lint,doc,py3

.github/workflows/remote_tests.yml

@@ -1,21 +0,0 @@
-name: Remote tests
-on: [push]
-jobs:
-  remote_tests_python:
-    name: Remote Python tests
-    runs-on: ubuntu-latest
-    env:
-      CIRCLE_BRANCH: ${{ github.ref }}
-      CIRCLE_JOB: remote_tests_python
-      CIRCLE_BUILD_NUM: ${{ github.run_number }}
-      DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-    steps:
-      - uses: actions/checkout@v2
-      - run: mkdir -m 700 -p ~/.ssh
-      - run: touch ~/.ssh/id_ed25519
-      - run: chmod 600 ~/.ssh/id_ed25519
-      - run: 'echo "$SSH_KEY" | base64 -d > ~/.ssh/id_ed25519'
-        shell: bash
-        env:
-          SSH_KEY: ${{ secrets.SSH_KEY }}
-      - run: ci_scripts/remote_tests_python.sh

.github/workflows/repl.yml

@@ -0,0 +1,32 @@
+# Manually triggered action to build a Windows repl.exe which users can
+# download to debug complex bugs.
+
+name: Build Windows REPL .exe
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_repl:
+    name: Build REPL example
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.50.0
+        override: true
+
+    - name: build
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --example repl --features repl,vendored
+
+    - name: Upload binary
+      uses: actions/upload-artifact@v2
+      with:
+        name: repl.exe
+        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -26,3 +26,5 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
+
+coverage/

CHANGELOG.md

@@ -1,5 +1,58 @@
 # Changelog 
 
+## 1.55.0
+
+- fix panic when receiving some HTML messages #2434
+
+- fix downloading some messages multiple times #2430
+
+- fix formatting of read receipt texts #2431
+
+- simplify SQL error handling #2415
+
+- explicit rust API for creating chats with blocked status #2282
+
+- debloat the binary by using less AsRef arguments #2425
+
+
+## 1.54.0
+
+- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
+
+- global search performance improvement #2364 #2365 #2366
+
+- improve SQLite performance with `PRAGMA synchronous=normal` #2382
+
+- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
+
+- python: list `requests` as a requirement #2390
+
+- fix creation of many delete jobs when being offline #2372
+
+- synchronize status between devices #2386
+
+- deaddrop (contact requests) chat improvements #2373
+
+- add "Forwarded:" to notification and chatlist summaries #2310
+
+- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
+
+- improve tests #2360 #2362 #2370 #2377 #2387
+
+- cleanup #2359 #2361 #2374 #2376 #2379 #2388
+
+
+## 1.53.0
+
+- fix sqlx performance regression #2355 2356
+
+- add a `ci_scripts/coverage.sh` #2333 #2334
+
+- refactorings and tests #2348 #2349 #2350
+
+- improve python bindings #2332 #2326
+
+
 ## 1.52.0
 
 - database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(deltachat)
+project(deltachat LANGUAGES C)
 
 find_program(CARGO cargo)
 
@@ -8,8 +8,18 @@ add_custom_command(
 	"target/release/libdeltachat.a"
 	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
-	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --package deltachat_ffi --release
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --release --no-default-features
+
+	# Build in `deltachat-ffi` directory instead of using
+	# `--package deltachat_ffi` to avoid feature resolver version
+	# "1" bug which makes `--no-default-features` affect only
+	# `deltachat`, but not `deltachat-ffi` package.
+	#
+	# We can't enable version "2" resolver [1] because it is not
+	# stable yet on rust 1.50.0.
+	#
+	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 
 add_custom_target(

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.52.0"
+version = "1.55.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,35 +12,37 @@ debug = 0
 lto = true
 
 [dependencies]
+deltachat_derive = { path = "./deltachat_derive" }
+
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.28"
-async-imap = "0.4.0"
+anyhow = "1.0.40"
+async-imap = "0.5.0"
 async-native-tls = { version = "0.3.3" }
 async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
-async-std-resolver = "0.19.5"
-async-std = { version = "~1.8.0", features = ["unstable"] }
+async-std-resolver = "0.20.3"
+async-std = { version = "~1.9.0", features = ["unstable"] }
 async-tar = "0.3.0"
-async-trait = "0.1.31"
-backtrace = "0.3.33"
+async-trait = "0.1.50"
+backtrace = "0.3.59"
 base64 = "0.13"
 bitflags = "1.1.0"
 byteorder = "1.3.1"
 charset = "0.1"
 chrono = "0.4.6"
-dirs = { version = "3.0.1", optional=true }
+dirs = { version = "3.0.2", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.0"
-futures = "0.3.4"
+escaper = "0.1.1"
+futures = "0.3.15"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 indexmap = "1.3.0"
 itertools = "0.10.0"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.51"
+libc = "0.2.95"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13.0"
+mailparse = "0.13.4"
 native-tls = "0.2.3"
 num_cpus = "1.13.0"
 num-derive = "0.3.0"
@@ -49,43 +51,45 @@ once_cell = "1.4.1"
 percent-encoding = "2.0"
 pgp = { version = "0.7.0", default-features = false }
 pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.18.1"
+quick-xml = "0.22.0"
+r2d2 = "0.8.9"
+r2d2_sqlite = "0.18.0"
 rand = "0.7.0"
-regex = "1.1.6"
+regex = "1.4.6"
+rusqlite = "0.25"
 rust-hsluv = "0.1.4"
-rustyline = { version = "4.1.0", optional = true }
+rustyline = { version = "8.2.0", optional = true }
 sanitize-filename = "0.3.0"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.3"
-sha2 = "0.9.0"
+sha-1 = "0.9.6"
+sha2 = "0.9.5"
 smallvec = "1.0.0"
-sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std-native-tls", "sqlite"] }
-# keep in sync with sqlx
-libsqlite3-sys = { version = "0.22.0", default-features = false, features = [ "pkg-config", "vcpkg", "bundled" ] }
-stop-token = { version = "0.1.1", features = ["unstable"] }
+stop-token = "0.2.0"
 strum = "0.20.0"
 strum_macros = "0.20.1"
 surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
-thiserror = "1.0.14"
+thiserror = "1.0.25"
 toml = "0.5.6"
-url = "2.1.1"
+url = "2.2.2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
+zip = "0.5.12"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
 criterion = "0.3"
 futures-lite = "1.7.0"
 log = "0.4.11"
-pretty_assertions = "0.6.1"
+pretty_assertions = "0.7.2"
 pretty_env_logger = "0.4.0"
-proptest = "0.10"
+proptest = "1.0"
 tempfile = "3.0"
 
 [workspace]
 members = [
   "deltachat-ffi",
+  "deltachat_derive",
 ]
 
 [[example]]
@@ -103,9 +107,17 @@ required-features = ["repl"]
 name = "create_account"
 harness = false
 
+[[bench]]
+name = "contacts"
+harness = false
+
+[[bench]]
+name = "search_msgs"
+harness = false
+
 [features]
 default = []
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
 nightly = ["pgp/nightly"]

README.md

@@ -2,7 +2,8 @@
 
 > Deltachat-core written in Rust 
 
-[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
+[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -111,11 +112,6 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
-[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
-[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
-[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
-[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
 ## Language bindings and frontend projects
 
 Language bindings are available for:

benches/contacts.rs

@@ -0,0 +1,39 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use tempfile::tempdir;
+
+async fn address_book_benchmark(n: u32, read_count: u32) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    let book = (0..n)
+        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .collect::<Vec<String>>()
+        .join("");
+
+    Contact::add_address_book(&context, &book).await.unwrap();
+
+    let query: Option<&str> = None;
+    for _ in 0..read_count {
+        Contact::get_all(&context, 0, query).await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 500 contacts", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+    });
+
+    c.bench_function("create 100 contacts and read it 1000 times", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -0,0 +1,29 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::context::Context;
+use std::path::Path;
+
+async fn search_benchmark(path: impl AsRef<Path>) {
+    let dbfile = path.as_ref();
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    for _ in 0..10u32 {
+        context.search_msgs(None, "hello").await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        c.bench_function("search hello", |b| {
+            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+        });
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.5 environment as the base environment 
-/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.5.*/bin/tox
-ln -s /opt/_internal/cpython-3.5.*/bin/devpi
-ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
-
-popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
-export PATH=/root/.cargo/bin:$PATH
-rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -xe
-export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
-export CIRCLE_BUILD_NUM=$USER
-export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
-export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
-
-time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -1,77 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-  push:
-
-env:
-  RUSTFLAGS: -Dwarnings
-
-jobs:
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly]
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      if: matrix.rust == 'nightly'
-      with:
-        command:  check
-        args: --all --bins --examples --tests
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
-
-    - name: tests ignored
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all --release -- --ignored
-
-  check_fmt:
-    name: Checking fmt and docs
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - uses: actions-rs/toolchain@v1
-      with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-
-    - name: fmt
-      run: cargo fmt --all -- --check
-
-#  clippy_check:
-#    name: Clippy check
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - uses: actions/checkout@v1
-#    - uses: actions-rs/toolchain@v1
-#      with:
-#          profile: minimal
-#          toolchain: nightly
-#          override: true
-#          components: clippy
-#
-#    - name: clippy
-#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -1,60 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library typically run in a docker
-# container that contains all library deps but should also work
-# outside if you have the dependencies installed on your system.
-
-set -e -x
-
-# Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
-
-# install core lib
-
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
-
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
-
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-if [ -n "$TESTS" ]; then
-
-    pushd python
-    # prepare a clean tox run
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-
-    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
-    # allows running of "liveconfig" tests but for speed reasons
-    # we run them only for the highest python version we support
-    
-    # we split out qr-tests run to minimize likelyness of flaky tests
-    # (some qr tests are pretty heavy in terms of send/received
-    # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
-    unset DCC_NEW_TMP_EMAIL
-    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
-    tox --workdir "$TOXWORKDIR" -e auditwheels
-    popd
-fi
-
-
-# if [ -n "$DOCS" ]; then
-#     echo -----------------------
-#     echo generating python docs
-#     echo -----------------------
-#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-# fi

ci_scripts/remote_tests_rust.sh

@@ -1,30 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-set -e
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-    export RUSTC_WRAPPER=\`which sccache\`
-    cd $BUILDDIR
-    export TARGET=x86_64-unknown-linux-gnu
-    export RUSTC_WRAPPER=sccache
-
-    bash ci_scripts/run-rust-test.sh
-_HERE
-

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.52.0"
+version = "1.55.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,9 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.6.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
+async-std = "1.9.0"
+anyhow = "1.0.40"
+thiserror = "1.0.25"
 rand = "0.7.3"
 
 [features]

deltachat-ffi/deltachat.h

@@ -1078,9 +1078,9 @@ int             dc_estimate_deletion_cnt    (dc_context_t* context, int from_ser
  * or badge counters eg. on the app-icon.
  * The list is already sorted and starts with the most recent fresh message.
  *
- * Messages belonging to muted chats are not returned,
- * as they should not be notified
- * and also a badge counters should not include messages of muted chats.
+ * Messages belonging to muted chats or to the deaddrop are not returned;
+ * these messages should not be notified
+ * and also badge counters should not include these messages.
  *
  * To get the number of fresh messages for a single chat, muted or not,
  * use dc_get_fresh_msg_cnt().
@@ -1104,7 +1104,8 @@ dc_array_t*     dc_get_fresh_msgs            (dc_context_t* context);
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
- * @param chat_id The chat ID of which all messages should be marked as being noticed.
+ * @param chat_id The chat ID of which all messages should be marked as being noticed
+ *     (this also works for the virtual chat ID DC_CHAT_ID_DEADDROP).
  */
 void            dc_marknoticed_chat          (dc_context_t* context, uint32_t chat_id);
 
@@ -1272,6 +1273,12 @@ uint32_t dc_get_chat_ephemeral_timer (dc_context_t* context, uint32_t chat_id);
  * search results may just hilite the corresponding messages and present a
  * prev/next button.
  *
+ * For global search, result is limited to 1000 messages,
+ * this allows incremental search done fast.
+ * So, when getting exactly 1000 results, the result may be truncated;
+ * the UIs may display sth. as "1000+ messages found" in this case.
+ * Chat search (if a chat_id is set) is not limited.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id ID of the chat to search messages in.
@@ -1587,13 +1594,22 @@ void            dc_marknoticed_contact       (dc_context_t* context, uint32_t co
 
 
 /**
- * Mark a message as _seen_, updates the IMAP state and
- * sends MDNs. If the message is not in a real chat (e.g. a contact request), the
- * message is only marked as NOTICED and no IMAP/MDNs is done.  See also
- * dc_marknoticed_chat().
+ * Mark messages as presented to the user.
+ * Typically, UIs call this function on scrolling through the chatlist,
+ * when the messages are presented at least for a little moment.
+ * The concrete action depends on the type of the chat and on the users settings
+ * (dc_msgs_presented() may be a better name therefore, but well :)
  *
- * Moreover, if messages belong to a chat with ephemeral messages enabled,
- * the ephemeral timer is started for these messages.
+ * - For normal chats, the IMAP state is updated, MDN is sent
+ *   (if dc_set_config()-options `mdns_enabled` is set)
+ *   and the internal state is changed to DC_STATE_IN_SEEN to reflect these actions.
+ *
+ * - For the deaddrop, no IMAP or MNDs is done
+ *   and the internal change is not changed therefore.
+ *   See also dc_marknoticed_chat().
+ *
+ * Moreover, timer is started for incoming ephemeral messages.
+ * This also happens for messages in the deaddrop.
  *
  * One #DC_EVENT_MSGS_NOTICED event is emitted per modified chat.
  *
@@ -2974,7 +2990,7 @@ char*           dc_chat_get_name             (const dc_chat_t* chat);
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return Path and file if the profile image, if any.
+ * @return Path and file of the profile image, if any.
  *     NULL otherwise.
  *     Must be released using dc_str_unref() after usage.
  */
@@ -3059,7 +3075,7 @@ int             dc_chat_is_device_talk       (const dc_chat_t* chat);
 
 
 /**
- * Check if messages can be sent to a give chat.
+ * Check if messages can be sent to a given chat.
  * This is not true e.g. for the deaddrop or for the device-talk, cmp. dc_chat_is_device_talk().
  *
  * Calling dc_send_msg() for these chats will fail
@@ -5619,6 +5635,11 @@ void dc_event_unref(dc_event_t* event);
 /// `%1$s` will be replaced by the number of weeks (always >1) the timer is set to.
 #define DC_STR_EPHEMERAL_WEEKS            96
 
+/// "Forwarded"
+///
+/// Used in message summary text for notifications and chatlist.
+#define DC_STR_FORWARDED                  97
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -21,6 +21,7 @@ use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
+use anyhow::Context as _;
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
@@ -130,12 +131,14 @@ pub unsafe extern "C" fn dc_set_config(
         return 0;
     }
     let ctx = &*context;
-    match config::Config::from_str(&to_string_lossy(key)) {
-        // When ctx.set_config() fails it already logged the error.
-        // TODO: Context::set_config() should not log this
+    let key = to_string_lossy(key);
+    match config::Config::from_str(&key) {
         Ok(key) => block_on(async move {
-            ctx.set_config(key, to_opt_string_lossy(value).as_deref())
+            let value = to_opt_string_lossy(value);
+            ctx.set_config(key, value.as_deref())
                 .await
+                .with_context(|| format!("Can't set {} to {:?}", key, value))
+                .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
         }),
         Err(_) => {
@@ -268,7 +271,7 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     let redirect = to_string_lossy(redirect);
 
     block_on(async move {
-        match oauth2::dc_get_oauth2_url(&ctx, addr, redirect).await {
+        match oauth2::dc_get_oauth2_url(&ctx, &addr, &redirect).await {
             Some(res) => res.strdup(),
             None => ptr::null_mut(),
         }
@@ -636,7 +639,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        chat::create_by_contact_id(&ctx, contact_id)
+        ChatId::create_for_contact(&ctx, contact_id)
             .await
             .log_err(ctx, "Failed to create chat from contact_id")
             .map(|id| id.to_u32())
@@ -656,11 +659,12 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        chat::get_by_contact_id(&ctx, contact_id)
+        ChatId::lookup_by_contact(&ctx, contact_id)
             .await
             .log_err(ctx, "Failed to get chat for contact_id")
+            .unwrap_or_default() // unwraps the Result
             .map(|id| id.to_u32())
-            .unwrap_or(0)
+            .unwrap_or(0) // unwraps the Option
     })
 }
 
@@ -1184,7 +1188,7 @@ pub unsafe extern "C" fn dc_search_msgs(
 
     block_on(async move {
         let arr = dc_array_t::from(
-            ctx.search_msgs(chat_id, to_string_lossy(query))
+            ctx.search_msgs(chat_id, &to_string_lossy(query))
                 .await
                 .unwrap_or_log_default(ctx, "Failed search_msgs")
                 .iter()
@@ -1233,7 +1237,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     };
 
     block_on(async move {
-        chat::create_group_chat(&ctx, protect, to_string_lossy(name))
+        chat::create_group_chat(&ctx, protect, &to_string_lossy(name))
             .await
             .log_err(ctx, "Failed to create group chat")
             .map(|id| id.to_u32())
@@ -1308,7 +1312,7 @@ pub unsafe extern "C" fn dc_set_chat_name(
     let ctx = &*context;
 
     block_on(async move {
-        chat::set_chat_name(&ctx, ChatId::new(chat_id), to_string_lossy(name))
+        chat::set_chat_name(&ctx, ChatId::new(chat_id), &to_string_lossy(name))
             .await
             .map(|_| 1)
             .unwrap_or_log_default(&ctx, "Failed to set chat name")
@@ -1505,8 +1509,7 @@ pub unsafe extern "C" fn dc_delete_msgs(
     let ctx = &*context;
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
 
-    block_on(message::delete_msgs(&ctx, &msg_ids));
-    info!(&ctx, "verbose (issue 2335): ffi called dc_delete_msgs()");
+    block_on(message::delete_msgs(&ctx, &msg_ids))
 }
 
 #[no_mangle]
@@ -1558,7 +1561,9 @@ pub unsafe extern "C" fn dc_markseen_msgs(
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ctx = &*context;
 
-    block_on(message::markseen_msgs(&ctx, msg_ids));
+    block_on(message::markseen_msgs(&ctx, msg_ids))
+        .log_err(ctx, "failed dc_markseen_msgs() call")
+        .ok();
 }
 
 #[no_mangle]
@@ -1637,7 +1642,7 @@ pub unsafe extern "C" fn dc_create_contact(
     let name = to_string_lossy(name);
 
     block_on(async move {
-        Contact::create(&ctx, name, to_string_lossy(addr))
+        Contact::create(&ctx, &name, &to_string_lossy(addr))
             .await
             .unwrap_or(0)
     })
@@ -1655,7 +1660,7 @@ pub unsafe extern "C" fn dc_add_address_book(
     let ctx = &*context;
 
     block_on(async move {
-        match Contact::add_address_book(&ctx, to_string_lossy(addr_book)).await {
+        match Contact::add_address_book(&ctx, &to_string_lossy(addr_book)).await {
             Ok(cnt) => cnt as libc::c_int,
             Err(_) => 0,
         }
@@ -1822,7 +1827,7 @@ pub unsafe extern "C" fn dc_imex(
 
     if let Some(param1) = to_opt_string_lossy(param1) {
         spawn(async move {
-            imex::imex(&ctx, what, &param1)
+            imex::imex(&ctx, what, param1.as_ref())
                 .await
                 .log_err(ctx, "IMEX failed")
         });
@@ -1843,7 +1848,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
     let ctx = &*context;
 
     block_on(async move {
-        match imex::has_backup(&ctx, to_string_lossy(dir)).await {
+        match imex::has_backup(&ctx, to_string_lossy(dir).as_ref()).await {
             Ok(res) => res.strdup(),
             Err(err) => {
                 // do not bubble up error to the user,
@@ -1924,7 +1929,7 @@ pub unsafe extern "C" fn dc_check_qr(
     let ctx = &*context;
 
     block_on(async move {
-        let lot = qr::check_qr(&ctx, to_string_lossy(qr)).await;
+        let lot = qr::check_qr(&ctx, &to_string_lossy(qr)).await;
         Box::into_raw(Box::new(lot))
     })
 }

deltachat_derive/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "deltachat_derive"
+version = "2.0.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+edition = "2018"
+license = "MPL-2.0"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "1.0.72"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -0,0 +1,47 @@
+#![recursion_limit = "128"]
+extern crate proc_macro;
+
+use crate::proc_macro::TokenStream;
+use quote::quote;
+
+// For now, assume (not check) that these macroses are applied to enum without
+// data.  If this assumption is violated, compiler error will point to
+// generated code, which is not very user-friendly.
+
+#[proc_macro_derive(ToSql)]
+pub fn to_sql_derive(input: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(input).unwrap();
+    let name = &ast.ident;
+
+    let gen = quote! {
+        impl rusqlite::types::ToSql for #name {
+            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+                let num = *self as i64;
+                let value = rusqlite::types::Value::Integer(num);
+                let output = rusqlite::types::ToSqlOutput::Owned(value);
+                std::result::Result::Ok(output)
+            }
+        }
+    };
+    gen.into()
+}
+
+#[proc_macro_derive(FromSql)]
+pub fn from_sql_derive(input: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(input).unwrap();
+    let name = &ast.ident;
+
+    let gen = quote! {
+        impl rusqlite::types::FromSql for #name {
+            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+                let inner = rusqlite::types::FromSql::column_result(col)?;
+                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
+                    Ok(value)
+                } else {
+                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
+                }
+            }
+        }
+    };
+    gen.into()
+}

examples/repl/cmdline.rs

@@ -13,6 +13,8 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
+use deltachat::error::Error;
+use deltachat::export_chat::export_chat_to_zip;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
@@ -34,7 +36,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 1 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM jobs;"))
+            .execute("DELETE FROM jobs;", paramsv![])
             .await
             .unwrap();
         println!("(1) Jobs reset.");
@@ -42,7 +44,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 2 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM acpeerstates;"))
+            .execute("DELETE FROM acpeerstates;", paramsv![])
             .await
             .unwrap();
         println!("(2) Peerstates reset.");
@@ -50,7 +52,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 4 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM keypairs;"))
+            .execute("DELETE FROM keypairs;", paramsv![])
             .await
             .unwrap();
         println!("(4) Private keypairs reset.");
@@ -58,34 +60,35 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 8 {
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM contacts WHERE id>9;"))
+            .execute("DELETE FROM contacts WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM chats WHERE id>9;"))
+            .execute("DELETE FROM chats WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM chats_contacts;"))
+            .execute("DELETE FROM chats_contacts;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM msgs WHERE id>9;"))
+            .execute("DELETE FROM msgs WHERE id>9;", paramsv![])
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query(
+            .execute(
                 "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-            ))
+                paramsv![],
+            )
             .await
             .unwrap();
         context
             .sql()
-            .execute(sqlx::query("DELETE FROM leftgrps;"))
+            .execute("DELETE FROM leftgrps;", paramsv![])
             .await
             .unwrap();
         println!("(8) Rest but server config reset.");
@@ -386,6 +389,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
+                 export-chat <chat-id> <destination-file>\n\
                  ===========================Contact requests==\n\
                  decidestartchat <msg-id>\n\
                  decideblock <msg-id>\n\
@@ -456,20 +460,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, &dir).await?;
+            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, arg1).await?;
+            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -525,9 +529,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = std::time::SystemTime::now()
-                .duration_since(time_start)
-                .unwrap_or_default();
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -604,7 +606,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
+            let time_start = std::time::SystemTime::now();
+            let msglist =
+                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
+
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -659,12 +665,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "{} messages.",
                 sel_chat.get_id().get_msg_cnt(&context).await?
             );
+
+            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
+            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
+
+            println!(
+                "{:?} to create this list, {:?} to mark all messages as noticed.",
+                time_needed, time_noticed_needed
+            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             let contact_id: u32 = arg1.parse()?;
-            let chat_id = chat::create_by_contact_id(&context, contact_id).await?;
+            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
@@ -898,10 +912,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None
             };
 
+            let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
+            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -1008,6 +1025,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
+        "export-chat" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            ensure!(!arg2.is_empty(), "Argument <destination file> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            // todo check if path is valid (dest dir exists) and ends in .zip
+            export_chat_to_zip(&context, chat_id, arg2).await;
+        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1045,7 +1069,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await;
+            message::markseen_msgs(&context, msg_ids).await?;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -1072,7 +1096,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, book).await?;
+                Contact::add_address_book(&context, &book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }

examples/repl/main.rs

@@ -26,8 +26,9 @@ use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
+use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
 
 mod cmdline;
@@ -168,7 +169,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 34] = [
+const CHAT_COMMANDS: [&str; 35] = [
     "listchats",
     "listarchived",
     "chat",
@@ -203,6 +204,7 @@ const CHAT_COMMANDS: [&str; 34] = [
     "protect",
     "unprotect",
     "delchat",
+    "export-chat",
 ];
 const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
@@ -237,7 +239,9 @@ const MISC_COMMANDS: [&str; 10] = [
 ];
 
 impl Hinter for DcHelper {
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
+    type Hint = String;
+
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -259,11 +263,10 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
-        if prompt == PROMPT {
+    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
+        if default {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -284,6 +287,7 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
+impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
@@ -317,8 +321,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }

examples/simple.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use deltachat::chat;
+use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
+    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);

python/README.rst

@@ -142,7 +142,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -151,9 +151,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps scripts/docker_coredeps
 
-This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -9,8 +9,6 @@ import subprocess
 import tempfile
 import textwrap
 import types
-from os.path import abspath
-from os.path import dirname as dn
 
 import cffi
 
@@ -50,6 +48,7 @@ def system_build_flags():
     flags.objs = []
     flags.incs = []
     flags.extra_link_args = []
+    return flags
 
 
 def extract_functions(flags):
@@ -168,11 +167,8 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
+        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()

python/src/deltachat/account.py

@@ -89,6 +89,22 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        log("=============== " + self.get_config("displayname") + " ===============")
+        cursor = 0
+        for name, val in self.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 

python/src/deltachat/direct_imap.py

@@ -47,8 +47,9 @@ def dc_account_extra_configure(account):
 
     except Exception as e:
         # Uncaught exceptions here would lead to a timeout without any note written to the log
-        account.log("=============================== CAN'T RESET ACCOUNT: ===============================")
-        account.log("===================", e, "===================")
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
 
 
 @deltachat.global_hookimpl
@@ -172,21 +173,6 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()

python/src/deltachat/message.py

@@ -86,6 +86,23 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
+    @props.with_doc
+    def html(self):
+        """html text of this messages (might be empty if not an html message). """
+        return from_dc_charpointer(
+            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
+
+    def has_html(self):
+        """return True if this message has an html part, False otherwise."""
+        return lib.dc_msg_has_html(self._dc_msg)
+
+    def set_html(self, html_text):
+        """set the html part of this message.
+
+        It is possible to have text and html part at the same time.
+        """
+        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
+
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -236,6 +253,20 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    @props.with_doc
+    def override_sender_name(self):
+        """the name that should be shown over the message instead of the contact display name.
+
+        Usually used to impersonate someone else.
+        """
+        return from_dc_charpointer(
+            lib.dc_msg_get_override_sender_name(self._dc_msg))
+
+    def set_override_sender_name(self, name):
+        """set different sender name for a message. """
+        lib.dc_msg_set_override_sender_name(
+            self._dc_msg, as_dc_charpointer(name))
+
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 

python/src/deltachat/testplugin.py

@@ -414,13 +414,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
-                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_accepted_chat(self, ac1, ac2):

python/tests/auditwheels.py

@@ -1,15 +1,17 @@
 
 import os
-import sys
+import platform
 import subprocess
+import sys
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
+    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_x86_64"])
+                 "--plat", "manylinux2014_" + arch])

python/tests/test_account.py

@@ -816,6 +816,48 @@ class TestOnlineAccount:
         assert open(msg.filename).read() == content
         assert msg.filename.endswith(basename)
 
+    def test_html_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        html_text = "<p>hello HTML world</p>"
+
+        lp.sec("ac1: prepare and send text message to ac2")
+        msg1 = chat.send_text("message0")
+        assert not msg1.has_html()
+        assert msg1.html == ""
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"
+        assert not msg2.has_html()
+        assert msg2.html == ""
+
+        lp.sec("ac1: prepare and send HTML+text message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message1")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+        assert msg1.has_html()
+        assert html_text in msg1.html
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
+        lp.sec("ac1: prepare and send HTML-only message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert "<p>" not in msg2.text
+        assert "hello HTML world" in msg2.text
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
@@ -991,16 +1033,75 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_moved_markseen(self, acfactory, lp):
+        """Test that message already moved to DeltaChat folder is marked as seen."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
+        ac2 = acfactory.get_online_configuring_account()
+        acfactory.wait_configure_and_start_io([ac1, ac2])
+        ac1.set_config("bcc_self", "1")
+
+        ac1.direct_imap.idle_start()
+        ac1.create_chat(ac2).send_text("Hello!")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.stop_io()
+
+        # Emulate moving of the message to DeltaChat folder by Sieve rule.
+        # mailcow server contains this rule by default.
+        ac1.direct_imap.conn.move(["*"], "DeltaChat")
+
+        ac1.direct_imap.select_folder("DeltaChat")
+        ac1.direct_imap.idle_start()
+        ac1.start_io()
+        ac1.direct_imap.idle_wait_for_seen()
+        ac1.direct_imap.idle_done()
+
+        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
+        flags = fetch[-1][b'FLAGS']
+        is_seen = b'\\Seen' in flags
+        assert is_seen
+
+    def test_message_override_sender_name(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        overridden_name = "someone else"
+
+        ac1.set_config("displayname", "ac1")
+
+        lp.sec("sending text message with overridden name from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_override_sender_name(overridden_name)
+        msg1.set_text("message1")
+        msg1 = chat.send_msg(msg1)
+        assert msg1.override_sender_name == overridden_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert msg2.override_sender_name == overridden_name
+
+        lp.sec("sending normal text message from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message2")
+        msg1 = chat.send_msg(msg1)
+        assert not msg1.override_sender_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message2"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert not msg2.override_sender_name
+
     @pytest.mark.parametrize("mvbox_move", [True, False])
     def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
         # Please only change this test if you are very sure that it will still catch the issues it catches now.
         # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
         ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
         ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
-        acfactory.wait_configure_and_start_io()
 
-        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
-        msg = ac2._evtracker.wait_next_incoming_message()
+        acfactory.wait_configure_and_start_io()
+        # Do not send BCC to self, we only want to test MDN on ac1.
+        ac1.set_config("bcc_self", "0")
 
         folder = "mvbox" if mvbox_move else "inbox"
         ac1.direct_imap.select_config_folder(folder)
@@ -1008,6 +1109,9 @@ class TestOnlineAccount:
         ac1.direct_imap.idle_start()
         ac2.direct_imap.idle_start()
 
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
         ac2.mark_seen_messages([msg])
 
         ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
@@ -1498,6 +1602,12 @@ class TestOnlineAccount:
         original_image_path = data.get_path("d.png")
         chat1.send_image(original_image_path)
 
+        # Add another 100KB file that ensures that the progress is smooth enough
+        path = tmpdir.join("attachment.txt")
+        with open(path, "w") as file:
+            file.truncate(100000)
+        chat1.send_file(path.strpath)
+
         def assert_account_is_proper(ac):
             contacts = ac.get_contacts(query="some1")
             assert len(contacts) == 1
@@ -1505,7 +1615,7 @@ class TestOnlineAccount:
             assert contact2.addr == "some1@example.org"
             chat2 = contact2.create_chat()
             messages = chat2.get_messages()
-            assert len(messages) == 2
+            assert len(messages) == 3
             assert messages[0].text == "msg1"
             assert messages[1].filemime == "image/png"
             assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
@@ -1633,7 +1743,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
         assert chat1.is_protected()
@@ -1664,6 +1774,29 @@ class TestOnlineAccount:
         assert msg.text == "world"
         assert msg.is_encrypted()
 
+        lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
+        qr = ac1.get_setup_contact_qr()
+
+        lp.sec("ac3: start QR-code based setup contact protocol")
+        ch = ac3.qr_setup_contact(qr)
+        assert ch.id >= 10
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+        lp.sec("ac1: add ac3 to verified group")
+        chat1.add_contact(ac3)
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.is_system_message()
+        assert not msg.error
+
+        lp.sec("ac2: send message and let ac3 read it")
+        chat2.send_text("hi")
+        # Skip system message about added member
+        ac3._evtracker.wait_next_incoming_message()
+        msg = ac3._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.is_encrypted()
+
     def test_set_get_contact_avatar(self, acfactory, data, lp):
         lp.sec("configuring ac1 and ac2")
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1867,6 +2000,47 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
+    def test_fetch_deleted_msg(self, acfactory, lp):
+        """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
+        hundreds of times, because uid_next was not updated.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2429.
+        """
+        ac1 = acfactory.get_one_online_account()
+        ac1.stop_io()
+
+        ac1.direct_imap.append("INBOX", """
+            From: alice <alice@example.org>
+            Subject: subj
+            To: bob@example.com
+            Chat-Version: 1.0
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Deleted message
+        """)
+        ac1.direct_imap.delete("1:*", expunge=False)
+        ac1.start_io()
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                break
+
+        # The message was downloaded once, now check that it's not downloaded again
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                pytest.fail("The same email was read twice")
+
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "INBOX: Idle entering wait-on-remote state" in ev.data2:
+                break  # DC is done with reading messages
+
     def test_send_receive_locations(self, acfactory, lp):
         now = datetime.utcnow()
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -2252,26 +2426,31 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
-    @pytest.mark.parametrize("folder,move,expected_destination,", [
-        ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
-        ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
-        ("Spam", False, "INBOX")  # ...emails are moved from the spam folder to the Inbox
+    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
+        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
+        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
     ])
     # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
     # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
-    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination):
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
         """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
         variant = folder + "-" + str(move) + "-" + expected_destination
         lp.sec("Testing variant " + variant)
         ac1 = acfactory.get_online_configuring_account(move=move)
         ac2 = acfactory.get_online_configuring_account()
+        ac1.set_config("inbox_watch", inbox_watch)
 
         acfactory.wait_configure(ac1)
         ac1.direct_imap.create_folder(folder)
 
         acfactory.wait_configure_and_start_io()
         # Wait until each folder was selected once and we are IDLEing:
-        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.stop_io()
 
         # Send a message to ac1 and move it to the mvbox:
@@ -2288,7 +2467,10 @@ class TestOnlineAccount:
         assert msg.text == "hello"
 
         # Wait until the message was moved (if at all) and we are IDLEing again:
-        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
         ac1.direct_imap.select_folder(expected_destination)
         assert len(ac1.direct_imap.get_all_messages()) == 1
         if folder != expected_destination:

python/tox.ini

@@ -1,7 +1,6 @@
 [tox]
-# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py37
+    py3
     lint
     auditwheels
 
@@ -46,10 +45,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    # With Python 3.7 and Sphinx 3.5.0, it throws an exception.
-    # Pin the version to the working one.
+    # Pin dependencies to the versions which actually work with Python 3.5.
     sphinx==3.4.3
-    breathe
+    breathe==4.28.0
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

scripts/README.md

@@ -30,8 +30,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    ci_scripts/manual_remote_tests.sh rust
-    ci_scripts/manual_remote_tests.sh python
+    scripts/manual_remote_tests.sh rust
+    scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -45,6 +45,10 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd ci_scripts  # where all CI things are 
+    cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
+
+Additionally, you can install qemu and build arm64 docker image:
+    apt-get install qemu binfmt-support qemu-user-static
+    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

scripts/ci_upload.sh

@@ -42,7 +42,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ci_scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
 rsync -avz \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   $WHEELHOUSEDIR \

scripts/coverage.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+
+if ! which grcov 2>/dev/null 1>&2; then
+    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
+    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
+    exit 1
+fi
+
+# Allow `-Z` flags without using nightly Rust.
+export RUSTC_BOOTSTRAP=1
+
+# We are using `-Zprofile` instead of source-based coverage [1]
+# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
+#
+# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
+# [2] https://github.com/mozilla/grcov/issues/595
+
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTDOCFLAGS="-Cpanic=abort"
+cargo clean
+cargo build
+cargo test
+
+grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2010_x86_64
+FROM quay.io/pypa/manylinux2014_aarch64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate 
+# Install a recent Perl, needed to install the openssl crate
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.6 environment as the base environment 
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+
+popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+curl "https://static.rust-lang.org/dist/rust-1.52.1-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
+cd "rust-1.52.1-$(uname -m)-unknown-linux-gnu"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+rustc --version
+cd ..
+rm -fr "rust-1.52.1-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -0,0 +1,21 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+# Install a recent Perl, needed to install the openssl crate
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_openssl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
+
+./Configure -de
+make
+make install

scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.6 environment as the base environment
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+
+popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "1.50.0-$(uname -m)-unknown-linux-gnu" -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf "/root/.rustup/toolchains/1.50.0-$(uname -m)-unknown-linux-gnu/share"

scripts/manual_remote_tests.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -xe
+
+JOB=${1:?need to specify 'rust' or 'python'}
+BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
+REPONAME="$(basename $(git rev-parse --show-toplevel))"
+
+time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"

scripts/remote_python_packaging.sh

@@ -15,6 +15,7 @@ git ls-files >.rsynclist
 # we seem to need .git for setuptools_scm versioning 
 find .git >>.rsynclist
 rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
 
 set +x
 
@@ -26,24 +27,41 @@ set +x
 # everything is terminated/cleaned up and we have no orphaned
 # useless still-running docker-containers consuming resources. 
 
-ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+for arch in "" "-arm64"; do
+
+ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
     set +x -e
     shopt -s huponexit
-    cd $BUILDDIR
+    cd ${BUILDDIR}${arch}
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
     set -x
 
     # run everything else inside docker 
     docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps${arch} scripts/run_all.sh
 
 _HERE
 
+done
+
 echo "--- Running $CIRCLE_JOB remotely"
 
-ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+echo "--- Building aarch64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
+
+echo "--- Building x86_64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+
 mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+
+# Wheels
+for arch in "" "-arm64"; do
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+done
+
+# Source packages
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
+
+# Documentation
 rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

scripts/remote_tests_python.sh

@@ -1,10 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILD_ID=${1:?specify build ID}
 
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -18,7 +17,7 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running $CIRCLE_JOB remotely"
+echo "--- Running Python tests remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
@@ -42,5 +41,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash ci_scripts/run-python-test.sh 
+    bash scripts/run-python-test.sh 
 _HERE

scripts/remote_tests_rust.sh

@@ -0,0 +1,29 @@
+#!/bin/bash 
+
+BUILD_ID=${1:?specify build ID}
+
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running Rust tests remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
+    cd $BUILDDIR
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash scripts/run-rust-test.sh
+_HERE
+

scripts/run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   ./run-integration-tests.sh -e py35
+#   scripts/run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   ./run-integration-tests.sh -e py35 -- -x
+#   scripts/run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}

scripts/run_all.sh

@@ -19,14 +19,17 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PATH=$PATH:/opt/python/cp36-cp36m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.5
-ln -s /opt/python/cp35-cp35m/bin/python3.5
+rm -f python3.6
 ln -s /opt/python/cp36-cp36m/bin/python3.6
+rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
+rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
+rm -f python3.9
+ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -40,7 +43,7 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
 popd
 
 

scripts/set_core_version.py

@@ -82,9 +82,10 @@ def main():
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit make sure to: ")
+    print("after commit, on master make sure to: ")
     print("")
-    print("   git tag {}".format(newversion))
+    print("   git tag -a {}".format(newversion))
+    print("   git push origin {}".format(newversion))
     print("")
 
 

src/blob.rs

@@ -1,5 +1,6 @@
 //! # Blob directory management
 
+use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 
@@ -7,8 +8,12 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::format_err;
+use anyhow::Context as _;
 use anyhow::Error;
+use image::DynamicImage;
 use image::GenericImageView;
+use image::ImageFormat;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
@@ -53,11 +58,11 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: impl AsRef<str>,
+        suggested_name: &str,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
         let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
@@ -132,18 +137,17 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: impl AsRef<Path>,
+        src: &Path,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file =
-            fs::File::open(src.as_ref())
-                .await
-                .map_err(|err| BlobError::CopyFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: String::from(""),
-                    src: src.as_ref().to_path_buf(),
-                    cause: err,
-                })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let mut src_file = fs::File::open(src)
+            .await
+            .map_err(|err| BlobError::CopyFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: String::from(""),
+                src: src.to_path_buf(),
+                cause: err,
+            })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
@@ -156,7 +160,7 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.as_ref().to_path_buf(),
+                src: src.to_path_buf(),
                 cause: err,
             });
         }
@@ -190,16 +194,13 @@ impl<'a> BlobObject<'a> {
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &Context,
-        src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject<'_>, BlobError> {
-        if src.as_ref().starts_with(context.get_blobdir()) {
+        context: &'a Context,
+        src: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        if src.starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.as_ref().starts_with("$BLOBDIR/") {
-            BlobObject::from_name(
-                context,
-                src.as_ref().to_str().unwrap_or_default().to_string(),
-            )
+        } else if src.starts_with("$BLOBDIR/") {
+            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -220,23 +221,22 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &Context,
-        path: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject, BlobError> {
-        let rel_path = path
-            .as_ref()
-            .strip_prefix(context.get_blobdir())
-            .map_err(|_| BlobError::WrongBlobdir {
-                blobdir: context.get_blobdir().to_path_buf(),
-                src: path.as_ref().to_path_buf(),
-            })?;
-        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+        context: &'a Context,
+        path: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let rel_path =
+            path.strip_prefix(context.get_blobdir())
+                .map_err(|_| BlobError::WrongBlobdir {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    src: path.to_path_buf(),
+                })?;
+        if !BlobObject::is_acceptible_blob_name(rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.as_ref().to_path_buf(),
+                blobname: path.to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.as_ref().to_path_buf(),
+            blobname: path.to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -380,7 +380,7 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
 
         let img_wh =
@@ -391,7 +391,15 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_AVATAR_SIZE,
             };
 
-        self.recode_to_size(context, blob_abs, img_wh).await
+        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
+        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
+        if let Some(new_name) = self
+            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
+            .await?
+        {
+            self.name = new_name;
+        }
+        Ok(())
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -410,30 +418,69 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_IMAGE_SIZE,
             };
 
-        self.recode_to_size(context, blob_abs, img_wh).await
+        if self
+            .recode_to_size(context, blob_abs, img_wh, None)
+            .await?
+            .is_some()
+        {
+            return Err(format_err!(
+                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
+            )
+            .into());
+        }
+        Ok(())
     }
 
     async fn recode_to_size(
         &self,
         context: &Context,
-        blob_abs: PathBuf,
-        img_wh: u32,
-    ) -> Result<(), BlobError> {
+        mut blob_abs: PathBuf,
+        mut img_wh: u32,
+        max_bytes: Option<usize>,
+    ) -> Result<Option<String>, BlobError> {
         let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
         let orientation = self.get_exif_orientation(context);
+        let mut encoded = Vec::new();
+        let mut changed_name = None;
 
-        let do_scale = img.width() > img_wh || img.height() > img_wh;
+        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
+            encoded.clear();
+            img.write_to(encoded, image::ImageFormat::Jpeg)?;
+            Ok(())
+        }
+        fn encode_img_exceeds_bytes(
+            context: &Context,
+            img: &DynamicImage,
+            max_bytes: Option<usize>,
+            encoded: &mut Vec<u8>,
+        ) -> anyhow::Result<bool> {
+            if let Some(max_bytes) = max_bytes {
+                encode_img(img, encoded)?;
+                if encoded.len() > max_bytes {
+                    info!(
+                        context,
+                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
+                        encoded.len(),
+                        img.width(),
+                        img.height(),
+                        max_bytes,
+                    );
+                    return Ok(true);
+                }
+            }
+            Ok(false)
+        }
+        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
+
+        let do_scale =
+            exceeds_width || encode_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
         let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
         if do_scale || do_rotate {
-            if do_scale {
-                img = img.thumbnail(img_wh, img_wh);
-            }
-
             if do_rotate {
                 img = match orientation {
                     Ok(90) => img.rotate90(),
@@ -443,14 +490,60 @@ impl<'a> BlobObject<'a> {
                 }
             }
 
-            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                cause: err.into(),
-            })?;
+            if do_scale {
+                if !exceeds_width {
+                    // The image is already smaller than img_wh, but exceeds max_bytes
+                    // We can directly start with trying to scale down to 2/3 of its current width
+                    img_wh = max(img.width(), img.height()) * 2 / 3
+                }
+
+                loop {
+                    let new_img = img.thumbnail(img_wh, img_wh);
+
+                    if encode_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
+                        if img_wh < 20 {
+                            return Err(format_err!(
+                                "Failed to scale image to below {}B",
+                                max_bytes.unwrap_or_default()
+                            )
+                            .into());
+                        }
+
+                        img_wh = img_wh * 2 / 3;
+                    } else {
+                        info!(
+                            context,
+                            "Final scaled-down image size: {}B ({}px)",
+                            encoded.len(),
+                            img_wh
+                        );
+                        break;
+                    }
+                }
+            }
+
+            // The file format is JPEG now, we may have to change the file extension
+            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
+                blob_abs = blob_abs.with_extension("jpg");
+                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
+                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
+                changed_name = Some(format!("$BLOBDIR/{}", file_name));
+            }
+
+            if encoded.is_empty() {
+                encode_img(&img, &mut encoded)?;
+            }
+
+            fs::write(&blob_abs, &encoded)
+                .await
+                .map_err(|err| BlobError::WriteFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                    cause: err.into(),
+                })?;
         }
 
-        Ok(())
+        Ok(changed_name)
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -514,14 +607,14 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
-    #[error("Sql: {0}")]
-    Sql(#[from] crate::sql::Error),
     #[error("{0}")]
     Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
+    use fs::File;
+
     use super::*;
 
     use crate::test_utils::TestContext;
@@ -626,13 +719,15 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
+        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
+            .await
+            .is_err());
         let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
@@ -643,7 +738,9 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -660,7 +757,9 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -715,4 +814,105 @@ mod tests {
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(&avatar_blob).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+
+        async fn file_size(path_buf: &PathBuf) -> u64 {
+            let file = File::open(path_buf).await.unwrap();
+            file.metadata().await.unwrap().len()
+        }
+
+        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
+
+        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
+            .await
+            .unwrap();
+        assert!(file_size(&avatar_blob).await <= 3000);
+        assert!(file_size(&avatar_blob).await > 2000);
+        let img = image::open(&avatar_blob).unwrap();
+        assert!(img.width() > 130);
+        assert_eq!(img.width(), img.height());
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
+        assert_eq!(
+            avatar_cfg,
+            avatar_src.with_extension("jpg").to_str().unwrap()
+        );
+
+        let img = image::open(avatar_cfg).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
 }

src/chatlist.rs

@@ -1,10 +1,7 @@
 //! # Chat list module
 
 use anyhow::{bail, ensure, Result};
-use async_std::prelude::*;
-use sqlx::Row;
 
-use crate::chat;
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
     Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
@@ -112,20 +109,23 @@ impl Chatlist {
 
         let mut add_archived_link_item = false;
 
-        let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .await
-                .unwrap_or_default()
-                .0
-        } else {
-            ChatId::new(0)
+        let process_row = |row: &rusqlite::Row| {
+            let chat_id: ChatId = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
         };
 
-        let process_row = |row: sqlx::Result<sqlx::sqlite::SqliteRow>| {
-            let row = row?;
-            let chat_id: ChatId = row.try_get(0)?;
-            let msg_id: MsgId = row.try_get(1).unwrap_or_default();
-            Ok((chat_id, msg_id))
+        let process_rows = |rows: rusqlite::MappedRows<_>| {
+            rows.collect::<std::result::Result<Vec<_>, _>>()
+                .map_err(Into::into)
+        };
+
+        let skip_id = if flag_for_forwarding {
+            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+                .await?
+                .unwrap_or_default()
+        } else {
+            ChatId::new(0)
         };
 
         // select with left join and minimum:
@@ -143,10 +143,10 @@ impl Chatlist {
         // tg do the same) for the deaddrop, however, they should
         // really be hidden, however, _currently_ the deaddrop is not
         // shown at all permanent in the chatlist.
-        let mut ids: Vec<_> = if let Some(query_contact_id) = query_contact_id {
+        let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.fetch(
-                sqlx::query("SELECT c.id, m.id
+            context.sql.query_map(
+                "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -160,9 +160,11 @@ impl Chatlist {
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;"
-                ).bind(MessageState::OutDraft).bind(query_contact_id).bind(ChatVisibility::Pinned)
-            ).await?.map(process_row).collect::<sqlx::Result<_>>().await?
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                process_row,
+                process_rows,
+            ).await?
         } else if flag_archived_only {
             // show archived chats
             // (this includes the archived device-chat; we could skip it,
@@ -170,9 +172,8 @@ impl Chatlist {
             // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.id, m.id
+                .query_map(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -187,13 +188,11 @@ impl Chatlist {
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    )
-                    .bind(MessageState::OutDraft),
+                    paramsv![MessageState::OutDraft],
+                    process_row,
+                    process_rows,
                 )
                 .await?
-                .map(process_row)
-                .collect::<sqlx::Result<_>>()
-                .await?
         } else if let Some(query) = query {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
@@ -207,9 +206,8 @@ impl Chatlist {
             let str_like_cmd = format!("%{}%", query);
             context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.id, m.id
+                .query_map(
+                    "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
@@ -224,27 +222,21 @@ impl Chatlist {
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    )
-                    .bind(MessageState::OutDraft)
-                    .bind(skip_id)
-                    .bind(str_like_cmd),
+                    paramsv![MessageState::OutDraft, skip_id, str_like_cmd],
+                    process_row,
+                    process_rows,
                 )
                 .await?
-                .map(process_row)
-                .collect::<sqlx::Result<_>>()
-                .await?
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .await
+                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                    .await?
                     .unwrap_or_default()
-                    .0
             } else {
                 ChatId::new(0)
             };
-
-            let mut ids: Vec<_> = context.sql.fetch(sqlx::query(
+            let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -259,15 +251,11 @@ impl Chatlist {
                    AND c.blocked=0
                    AND NOT c.archived=?3
                  GROUP BY c.id
-                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;"
-            )
-              .bind(MessageState::OutDraft)
-              .bind(skip_id)
-              .bind(ChatVisibility::Archived)
-              .bind(sort_id_up)
-              .bind(ChatVisibility::Pinned)
-            ).await?.map(process_row).collect::<sqlx::Result<_>>().await?;
-
+                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                process_row,
+                process_rows,
+            ).await?;
             if !flag_no_specials {
                 if let Some(last_deaddrop_fresh_msg_id) =
                     get_last_deaddrop_fresh_msg(context).await?
@@ -410,9 +398,10 @@ impl Chatlist {
 pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
     let count = context
         .sql
-        .count(sqlx::query(
+        .count(
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
-        ))
+            paramsv![],
+        )
         .await?;
     Ok(count)
 }
@@ -422,16 +411,19 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>>
     // sufficient as there are typically only few fresh messages.
     let id = context
         .sql
-        .query_get_value(sqlx::query(concat!(
-            "SELECT m.id",
-            " FROM msgs m",
-            " LEFT JOIN chats c",
-            "        ON c.id=m.chat_id",
-            " WHERE m.state=10",
-            "   AND m.hidden=0",
-            "   AND c.blocked=2",
-            " ORDER BY m.timestamp DESC, m.id DESC;"
-        )))
+        .query_get_value(
+            concat!(
+                "SELECT m.id",
+                " FROM msgs m",
+                " LEFT JOIN chats c",
+                "        ON c.id=m.chat_id",
+                " WHERE m.state=10",
+                "   AND m.hidden=0",
+                "   AND c.blocked=2",
+                " ORDER BY m.timestamp DESC, m.id DESC;"
+            ),
+            paramsv![],
+        )
         .await?;
     Ok(id)
 }

src/color.rs

@@ -8,8 +8,8 @@ use hsluv::hsluv_to_rgb;
 use sha1::{Digest, Sha1};
 
 /// Converts an identifier to Hue angle.
-fn str_to_angle(s: impl AsRef<str>) -> f64 {
-    let bytes = s.as_ref().as_bytes();
+fn str_to_angle(s: &str) -> f64 {
+    let bytes = s.as_bytes();
     let result = Sha1::digest(bytes);
     let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
         + 256 * result.get(1).map_or(0, |&x| u16::from(x));
@@ -31,7 +31,7 @@ fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
 ///
 /// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
 /// half (50.0) to make colors suitable both for light and dark theme.
-pub(crate) fn str_to_color(s: impl AsRef<str>) -> u32 {
+pub(crate) fn str_to_color(s: &str) -> u32 {
     rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
 }
 

src/config.rs

@@ -242,14 +242,14 @@ impl Context {
         match key {
             Config::Selfavatar => {
                 self.sql
-                    .execute(sqlx::query("UPDATE contacts SET selfavatar_sent=0;"))
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![])
                     .await?;
                 self.sql
                     .set_raw_config_bool("attach_selfavatar", true)
                     .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(self, value).await?;
+                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql.set_raw_config(key, Some(blob.as_name())).await?;
                         Ok(())
@@ -305,7 +305,7 @@ impl Context {
         }
     }
 
-    pub async fn set_config_bool(&self, key: Config, value: bool) -> crate::sql::Result<()> {
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
         self.set_config(key, if value { Some("1") } else { None })
             .await?;
         Ok(())
@@ -331,12 +331,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::BALANCED_AVATAR_SIZE;
     use crate::test_utils::TestContext;
-    use image::GenericImageView;
     use num_traits::FromPrimitive;
-    use std::fs::File;
-    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -350,82 +346,6 @@ mod tests {
         );
     }
 
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;

src/configure.rs

@@ -449,7 +449,7 @@ async fn get_autoconfig(
 ) -> Option<Vec<ServerParams>> {
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        format!(
+        &format!(
             "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
             param_domain, param_addr_urlencoded
         ),
@@ -464,7 +464,7 @@ async fn get_autoconfig(
     if let Ok(res) = moz_autoconfigure(
         ctx,
         // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-        format!(
+        &format!(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
         ),
@@ -503,7 +503,7 @@ async fn get_autoconfig(
     // always SSL for Thunderbird's database
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
         param,
     )
     .await

src/configure/auto_mozilla.rs

@@ -251,10 +251,10 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
 
 pub(crate) async fn moz_autoconfigure(
     context: &Context,
-    url: impl AsRef<str>,
+    url: &str,
     param_in: &LoginParam,
 ) -> Result<Vec<ServerParams>, Error> {
-    let xml_raw = read_url(context, url.as_ref()).await?;
+    let xml_raw = read_url(context, url).await?;
 
     let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/server_params.rs

@@ -25,7 +25,7 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
@@ -42,7 +42,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
         if self.hostname.is_empty() {
             self.hostname = param_domain.to_string();
@@ -62,7 +62,7 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+    fn expand_ports(mut self) -> Vec<ServerParams> {
         // Try to infer port from socket security.
         if self.port == 0 {
             self.port = match self.socket {
@@ -160,5 +160,37 @@ mod tests {
                 username: "foobar".to_string(),
             }],
         );
+
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Automatic,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string()
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string()
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -1,4 +1,5 @@
 //! # Constants
+use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
@@ -15,9 +16,10 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(i8)]
 pub enum Blocked {
@@ -32,7 +34,9 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -46,7 +50,9 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -59,7 +65,9 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -73,7 +81,9 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -133,10 +143,11 @@ pub const DC_CHAT_ID_LAST_SPECIAL: ChatId = ChatId::new(9);
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     IntoStaticStr,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(u32)]
 pub enum Chattype {
@@ -247,9 +258,10 @@ pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     Serialize,
     Deserialize,
-    sqlx::Type,
 )]
 #[repr(u32)]
 pub enum Viewtype {
@@ -310,16 +322,6 @@ impl Default for Viewtype {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -328,3 +330,100 @@ pub enum KeyType {
     Public = 0,
     Private = 1,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_chattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Chattype::Undefined, Chattype::default());
+        assert_eq!(Chattype::Undefined, Chattype::from_i32(0).unwrap());
+        assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
+        assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
+        assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+    }
+
+    #[test]
+    fn test_keygentype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyGenType::Default, KeyGenType::default());
+        assert_eq!(KeyGenType::Default, KeyGenType::from_i32(0).unwrap());
+        assert_eq!(KeyGenType::Rsa2048, KeyGenType::from_i32(1).unwrap());
+        assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_keytype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap());
+        assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_showemails_values() {
+        // values may be written to disk and must not change
+        assert_eq!(ShowEmails::Off, ShowEmails::default());
+        assert_eq!(ShowEmails::Off, ShowEmails::from_i32(0).unwrap());
+        assert_eq!(
+            ShowEmails::AcceptedContacts,
+            ShowEmails::from_i32(1).unwrap()
+        );
+        assert_eq!(ShowEmails::All, ShowEmails::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_blocked_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Blocked::Not, Blocked::default());
+        assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
+        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_mediaquality_values() {
+        // values may be written to disk and must not change
+        assert_eq!(MediaQuality::Balanced, MediaQuality::default());
+        assert_eq!(MediaQuality::Balanced, MediaQuality::from_i32(0).unwrap());
+        assert_eq!(MediaQuality::Worse, MediaQuality::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_videochattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(VideochatType::Unknown, VideochatType::default());
+        assert_eq!(VideochatType::Unknown, VideochatType::from_i32(0).unwrap());
+        assert_eq!(
+            VideochatType::BasicWebrtc,
+            VideochatType::from_i32(1).unwrap()
+        );
+        assert_eq!(VideochatType::Jitsi, VideochatType::from_i32(2).unwrap());
+    }
+}

src/contact.rs

@@ -1,13 +1,13 @@
 //! Contacts module
-use std::convert::TryFrom;
+
+use std::convert::{TryFrom, TryInto};
 
 use anyhow::{bail, ensure, format_err, Result};
 use async_std::path::PathBuf;
-use async_std::prelude::*;
+use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
-use sqlx::Row;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -79,7 +79,7 @@ pub struct Contact {
 
 /// Possible origins of a contact.
 #[derive(
-    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromPrimitive, ToPrimitive, sqlx::Type,
+    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, FromPrimitive, ToPrimitive, FromSql, ToSql,
 )]
 #[repr(u32)]
 pub enum Origin {
@@ -175,30 +175,36 @@ pub enum VerifiedStatus {
 }
 
 impl Contact {
-    pub async fn load_from_db(context: &Context, contact_id: u32) -> crate::sql::Result<Self> {
-        let row = context
+    pub async fn load_from_db(context: &Context, contact_id: u32) -> Result<Self> {
+        let mut contact = context
             .sql
-            .fetch_one(
-                sqlx::query(
-                    "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param, c.status
+            .query_row(
+                "SELECT c.name, c.addr, c.origin, c.blocked, c.authname, c.param, c.status
                FROM contacts c
               WHERE c.id=?;",
-                )
-                .bind(contact_id),
+                paramsv![contact_id as i32],
+                |row| {
+                    let name: String = row.get(0)?;
+                    let addr: String = row.get(1)?;
+                    let origin: Origin = row.get(2)?;
+                    let blocked: Option<bool> = row.get(3)?;
+                    let authname: String = row.get(4)?;
+                    let param: String = row.get(5)?;
+                    let status: Option<String> = row.get(6)?;
+                    let contact = Self {
+                        id: contact_id,
+                        name,
+                        authname,
+                        addr,
+                        blocked: blocked.unwrap_or_default(),
+                        origin,
+                        param: param.parse().unwrap_or_default(),
+                        status: status.unwrap_or_default(),
+                    };
+                    Ok(contact)
+                },
             )
             .await?;
-
-        let mut contact = Contact {
-            id: contact_id,
-            name: row.try_get(0)?,
-            authname: row.try_get(4)?,
-            addr: row.try_get(1)?,
-            blocked: row.try_get::<Option<i32>, _>(3)?.unwrap_or_default() != 0,
-            origin: row.try_get(2)?,
-            param: row.try_get::<String, _>(5)?.parse().unwrap_or_default(),
-            status: row.try_get::<Option<String>, _>(6)?.unwrap_or_default(),
-        };
-
         if contact_id == DC_CONTACT_ID_SELF {
             contact.name = stock_str::self_msg(context).await;
             contact.addr = context
@@ -213,7 +219,6 @@ impl Contact {
             contact.name = stock_str::device_messages(context).await;
             contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
         }
-
         Ok(contact)
     }
 
@@ -249,21 +254,14 @@ impl Contact {
     /// a bunch of addresses.
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
-    pub async fn create(
-        context: &Context,
-        name: impl AsRef<str>,
-        addr: impl AsRef<str>,
-    ) -> Result<u32> {
+    pub async fn create(context: &Context, name: &str, addr: &str) -> Result<u32> {
         let name = improve_single_line_input(name);
-        ensure!(
-            !addr.as_ref().is_empty(),
-            "Cannot create contact with empty address"
-        );
+        ensure!(!addr.is_empty(), "Cannot create contact with empty address");
 
-        let (name, addr) = sanitize_name_and_addr(name, addr);
+        let (name, addr) = sanitize_name_and_addr(&name, addr);
 
         let (contact_id, sth_modified) =
-            Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
+            Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
         match sth_modified {
             Modifier::None => {}
@@ -285,10 +283,8 @@ impl Contact {
         if context
             .sql
             .execute(
-                sqlx::query("UPDATE msgs SET state=? WHERE from_id=? AND state=?;")
-                    .bind(MessageState::InNoticed)
-                    .bind(id as i32)
-                    .bind(MessageState::InFresh),
+                "UPDATE msgs SET state=? WHERE from_id=? AND state=?;",
+                paramsv![MessageState::InNoticed, id as i32, MessageState::InFresh],
             )
             .await
             .is_ok()
@@ -322,18 +318,16 @@ impl Contact {
         let id = context
             .sql
             .query_get_value(
-                sqlx::query(
-                    "SELECT id FROM contacts \
-                WHERE addr=?1 COLLATE NOCASE \
-                AND id>?2 AND origin>=?3 AND blocked=0;",
-                )
-                .bind(addr_normalized)
-                .bind(DC_CONTACT_ID_LAST_SPECIAL)
-                .bind(min_origin),
+                "SELECT id FROM contacts \
+            WHERE addr=?1 COLLATE NOCASE \
+            AND id>?2 AND origin>=?3 AND blocked=0;",
+                paramsv![
+                    addr_normalized,
+                    DC_CONTACT_ID_LAST_SPECIAL as i32,
+                    min_origin as u32,
+                ],
             )
-            .await?
-            .unwrap_or_default();
-
+            .await?;
         Ok(id)
     }
 
@@ -364,19 +358,16 @@ impl Contact {
     /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
     pub(crate) async fn add_or_lookup(
         context: &Context,
-        name: impl AsRef<str>,
-        addr: impl AsRef<str>,
+        name: &str,
+        addr: &str,
         mut origin: Origin,
     ) -> Result<(u32, Modifier)> {
         let mut sth_modified = Modifier::None;
 
-        ensure!(
-            !addr.as_ref().is_empty(),
-            "Can not add_or_lookup empty address"
-        );
+        ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
-        let addr = addr_normalize(addr.as_ref()).to_string();
+        let addr = addr_normalize(addr).to_string();
         let addr_self = context
             .get_config(Config::ConfiguredAddr)
             .await?
@@ -391,16 +382,12 @@ impl Contact {
                 context,
                 "Bad address \"{}\" for contact \"{}\".",
                 addr,
-                if !name.as_ref().is_empty() {
-                    name.as_ref()
-                } else {
-                    "<unset>"
-                },
+                if !name.is_empty() { name } else { "<unset>" },
             );
             bail!("Bad address supplied: {:?}", addr);
         }
 
-        let mut name = name.as_ref();
+        let mut name = name;
         #[allow(clippy::collapsible_if)]
         if origin <= Origin::OutgoingTo {
             // The user may accidentally have written to a "noreply" address with another MUA:
@@ -433,23 +420,21 @@ impl Contact {
 
         if let Ok((id, row_name, row_addr, row_origin, row_authname)) = context
             .sql
-            .fetch_one(
-                sqlx::query(
-                    "SELECT id, name, addr, origin, authname \
-                    FROM contacts WHERE addr=? COLLATE NOCASE;",
-                )
-                .bind(addr.to_string()),
+            .query_row(
+                "SELECT id, name, addr, origin, authname \
+            FROM contacts WHERE addr=? COLLATE NOCASE;",
+                paramsv![addr.to_string()],
+                |row| {
+                    let row_id: isize = row.get(0)?;
+                    let row_name: String = row.get(1)?;
+                    let row_addr: String = row.get(2)?;
+                    let row_origin: Origin = row.get(3)?;
+                    let row_authname: String = row.get(4)?;
+
+                    Ok((row_id, row_name, row_addr, row_origin, row_authname))
+                },
             )
             .await
-            .and_then(|row| {
-                let row_id = row.try_get(0)?;
-                let row_name: String = row.try_get(1)?;
-                let row_addr: String = row.try_get(2)?;
-                let row_origin: Origin = row.try_get(3)?;
-                let row_authname: String = row.try_get(4)?;
-
-                Ok((row_id, row_name, row_addr, row_origin, row_authname))
-            })
         {
             let update_name = manual && name != row_name;
             let update_authname = !manual
@@ -458,7 +443,8 @@ impl Contact {
                 && (origin >= row_origin
                     || origin == Origin::IncomingUnknownFrom
                     || row_authname.is_empty());
-            row_id = id;
+
+            row_id = u32::try_from(id)?;
             if origin as i32 >= row_origin as i32 && addr != row_addr {
                 update_addr = true;
             }
@@ -469,36 +455,39 @@ impl Contact {
                     row_name
                 };
 
-                let query = sqlx::query(
-                    "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
-                )
-                .bind(&new_name)
-                .bind(if update_addr {
-                    addr.to_string()
-                } else {
-                    row_addr
-                })
-                .bind(if origin > row_origin {
-                    origin
-                } else {
-                    row_origin
-                })
-                .bind(if update_authname {
-                    name.to_string()
-                } else {
-                    row_authname
-                })
-                .bind(row_id);
-
-                context.sql.execute(query).await.ok();
+                context
+                    .sql
+                    .execute(
+                        "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
+                        paramsv![
+                            new_name,
+                            if update_addr {
+                                addr.to_string()
+                            } else {
+                                row_addr
+                            },
+                            if origin > row_origin {
+                                origin
+                            } else {
+                                row_origin
+                            },
+                            if update_authname {
+                                name.to_string()
+                            } else {
+                                row_authname
+                            },
+                            row_id
+                        ],
+                    )
+                    .await
+                    .ok();
 
                 if update_name {
                     // Update the contact name also if it is used as a group name.
                     // This is one of the few duplicated data, however, getting the chat list is easier this way.
-                    let chat_id = context.sql.query_get_value::<_, u32>(
-                        sqlx::query(
-                            "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)"
-                        ).bind(Chattype::Single).bind(row_id)
+                    let chat_id: Option<i32> = context.sql.query_get_value(
+                        "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                        paramsv![Chattype::Single, isize::try_from(row_id)?]
                     ).await?;
                     if let Some(chat_id) = chat_id {
                         let contact = Contact::get_by_id(context, row_id as u32).await?;
@@ -506,10 +495,8 @@ impl Contact {
                         match context
                             .sql
                             .execute(
-                                sqlx::query("UPDATE chats SET name=?1 WHERE id=?2 AND name!=?3")
-                                    .bind(&chat_name)
-                                    .bind(chat_id)
-                                    .bind(&chat_name),
+                                "UPDATE chats SET name=?1 WHERE id=?2 AND name!=?3",
+                                paramsv![chat_name, chat_id, chat_name],
                             )
                             .await
                         {
@@ -517,8 +504,9 @@ impl Contact {
                             Ok(count) => {
                                 if count > 0 {
                                     // Chat name updated
-                                    context
-                                        .emit_event(EventType::ChatModified(ChatId::new(chat_id)));
+                                    context.emit_event(EventType::ChatModified(ChatId::new(
+                                        chat_id.try_into()?,
+                                    )));
                                 }
                             }
                         }
@@ -530,29 +518,28 @@ impl Contact {
             let update_name = manual;
             let update_authname = !manual;
 
-            if context
+            if let Ok(new_row_id) = context
                 .sql
-                .execute(
-                    sqlx::query(
-                        "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
-                    )
-                    .bind(if update_name {
-                        name.to_string()
-                    } else {
-                        "".to_string()
-                    })
-                    .bind(&addr)
-                    .bind(origin)
-                    .bind(if update_authname {
-                        name.to_string()
-                    } else {
-                        "".to_string()
-                    }),
+                .insert(
+                    "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
+                    paramsv![
+                        if update_name {
+                            name.to_string()
+                        } else {
+                            "".to_string()
+                        },
+                        addr,
+                        origin,
+                        if update_authname {
+                            name.to_string()
+                        } else {
+                            "".to_string()
+                        }
+                    ],
                 )
                 .await
-                .is_ok()
             {
-                row_id = context.sql.get_rowid("contacts", "addr", &addr).await?;
+                row_id = u32::try_from(new_row_id)?;
                 sth_modified = Modifier::Created;
                 info!(context, "added contact id={} addr={}", row_id, &addr);
             } else {
@@ -560,7 +547,7 @@ impl Contact {
             }
         }
 
-        Ok((u32::try_from(row_id)?, sth_modified))
+        Ok((row_id, sth_modified))
     }
 
     /// Add a number of contacts.
@@ -580,13 +567,13 @@ impl Contact {
     /// The `addr_book` is a multiline string in the format `Name one\nAddress one\nName two\nAddress two`.
     ///
     /// Returns the number of modified contacts.
-    pub async fn add_address_book(context: &Context, addr_book: impl AsRef<str>) -> Result<usize> {
+    pub async fn add_address_book(context: &Context, addr_book: &str) -> Result<usize> {
         let mut modify_cnt = 0;
 
-        for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
+        for (name, addr) in split_address_book(addr_book).into_iter() {
             let (name, addr) = sanitize_name_and_addr(name, addr);
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, &addr, Origin::AddressBook).await {
+            match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
                         context,
@@ -632,19 +619,11 @@ impl Contact {
         let flag_add_self = (listflags & DC_GCL_ADD_SELF) != 0;
 
         if flag_verified_only || query.is_some() {
-            let s3str_like_cmd = format!(
-                "%{}%",
-                query
-                    .as_ref()
-                    .map(|s| s.as_ref().to_string())
-                    .unwrap_or_default()
-            );
-
-            let mut rows = context
+            let s3str_like_cmd = format!("%{}%", query.as_ref().map(|s| s.as_ref()).unwrap_or(""));
+            context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.id FROM contacts c \
+                .query_map(
+                    "SELECT c.id FROM contacts c \
                  LEFT JOIN acpeerstates ps ON c.addr=ps.addr  \
                  WHERE c.addr!=?1 \
                  AND c.id>?2 \
@@ -653,19 +632,23 @@ impl Contact {
                  AND (iif(c.name='',c.authname,c.name) LIKE ?4 OR c.addr LIKE ?5) \
                  AND (1=?6 OR LENGTH(ps.verified_key_fingerprint)!=0)  \
                  ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
-                    )
-                    .bind(&self_addr)
-                    .bind(DC_CONTACT_ID_LAST_SPECIAL)
-                    .bind(Origin::IncomingReplyTo)
-                    .bind(&s3str_like_cmd)
-                    .bind(&s3str_like_cmd)
-                    .bind(if flag_verified_only { 0i32 } else { 1i32 }),
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL as i32,
+                        Origin::IncomingReplyTo,
+                        s3str_like_cmd,
+                        s3str_like_cmd,
+                        if flag_verified_only { 0i32 } else { 1i32 },
+                    ],
+                    |row| row.get::<_, i32>(0),
+                    |ids| {
+                        for id in ids {
+                            ret.push(id? as u32);
+                        }
+                        Ok(())
+                    },
                 )
-                .await?
-                .map(|row| row?.try_get(0));
-            while let Some(id) = rows.next().await {
-                ret.push(id?);
-            }
+                .await?;
 
             let self_name = context
                 .get_config(Config::Displayname)
@@ -686,27 +669,29 @@ impl Contact {
         } else {
             add_self = true;
 
-            let mut rows = context
+            context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT id FROM contacts
+                .query_map(
+                    "SELECT id FROM contacts
                  WHERE addr!=?1
                  AND id>?2
                  AND origin>=?3
                  AND blocked=0
                  ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                    )
-                    .bind(self_addr)
-                    .bind(DC_CONTACT_ID_LAST_SPECIAL)
-                    .bind(Origin::IncomingReplyTo),
+                    paramsv![
+                        self_addr,
+                        DC_CONTACT_ID_LAST_SPECIAL as i32,
+                        Origin::IncomingReplyTo
+                    ],
+                    |row| row.get::<_, i32>(0),
+                    |ids| {
+                        for id in ids {
+                            ret.push(id? as u32);
+                        }
+                        Ok(())
+                    },
                 )
-                .await?
-                .map(|row| row?.try_get(0));
-
-            while let Some(id) = rows.next().await {
-                ret.push(id?);
-            }
+                .await?;
         }
 
         if flag_add_self && add_self {
@@ -722,38 +707,38 @@ impl Contact {
     // from the users perspective,
     // there is not much difference in an email- and a mailinglist-address)
     async fn update_blocked_mailinglist_contacts(context: &Context) -> Result<()> {
-        let mut rows = context
+        let blocked_mailinglists = context
             .sql
-            .fetch(
-                sqlx::query("SELECT name, grpid FROM chats WHERE type=? AND blocked=?;")
-                    .bind(Chattype::Mailinglist)
-                    .bind(Blocked::Manually),
+            .query_map(
+                "SELECT name, grpid FROM chats WHERE type=? AND blocked=?;",
+                paramsv![Chattype::Mailinglist, Blocked::Manually],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)),
+                |rows| {
+                    rows.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
             )
             .await?;
-
-        while let Some(row) = rows.next().await {
-            let row = row?;
-            let name = row.try_get::<String, _>(0)?;
-            let grpid = row.try_get::<String, _>(1)?;
-
+        for (name, grpid) in blocked_mailinglists {
             if !context
                 .sql
-                .exists(sqlx::query("SELECT COUNT(id) FROM contacts WHERE addr=?;").bind(&grpid))
+                .exists(
+                    "SELECT COUNT(id) FROM contacts WHERE addr=?;",
+                    paramsv![grpid],
+                )
                 .await?
             {
                 context
                     .sql
-                    .execute(sqlx::query("INSERT INTO contacts (addr) VALUES (?);").bind(&grpid))
+                    .execute("INSERT INTO contacts (addr) VALUES (?);", paramsv![grpid])
                     .await?;
             }
             // always do an update in case the blocking is reset or name is changed
             context
                 .sql
                 .execute(
-                    sqlx::query("UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?;")
-                        .bind(name)
-                        .bind(Origin::MailinglistAddress)
-                        .bind(&grpid),
+                    "UPDATE contacts SET name=?, origin=?, blocked=1 WHERE addr=?;",
+                    paramsv![name, Origin::MailinglistAddress, grpid],
                 )
                 .await?;
         }
@@ -764,8 +749,8 @@ impl Contact {
         let count = context
             .sql
             .count(
-                sqlx::query("SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0")
-                    .bind(DC_CONTACT_ID_LAST_SPECIAL),
+                "SELECT COUNT(*) FROM contacts WHERE id>? AND blocked!=0",
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL],
             )
             .await?;
         Ok(count as usize)
@@ -782,16 +767,16 @@ impl Contact {
 
         let list = context
             .sql
-            .fetch(
-                sqlx::query(
+            .query_map(
                 "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
-                ).bind(DC_CONTACT_ID_LAST_SPECIAL)
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
+                |row| row.get::<_, u32>(0),
+                |ids| {
+                    ids.collect::<std::result::Result<Vec<_>, _>>()
+                        .map_err(Into::into)
+                },
             )
-            .await?
-            .map(|row| row?.try_get::<u32, _>(0))
-            .collect::<sqlx::Result<Vec<_>>>()
             .await?;
-
         Ok(list)
     }
 
@@ -844,14 +829,14 @@ impl Contact {
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
                     cat_fingerprint(
                         &mut ret,
-                        peerstate.addr.clone(),
+                        &peerstate.addr,
                         &fingerprint_other_verified,
                         &fingerprint_other_unverified,
                     );
                 } else {
                     cat_fingerprint(
                         &mut ret,
-                        peerstate.addr.clone(),
+                        &peerstate.addr,
                         &fingerprint_other_verified,
                         &fingerprint_other_unverified,
                     );
@@ -878,8 +863,8 @@ impl Contact {
         let count_contacts = context
             .sql
             .count(
-                sqlx::query("SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?;")
-                    .bind(contact_id),
+                "SELECT COUNT(*) FROM chats_contacts WHERE contact_id=?;",
+                paramsv![contact_id as i32],
             )
             .await?;
 
@@ -887,9 +872,8 @@ impl Contact {
             context
                 .sql
                 .count(
-                    sqlx::query("SELECT COUNT(*) FROM msgs WHERE from_id=? OR to_id=?;")
-                        .bind(contact_id)
-                        .bind(contact_id),
+                    "SELECT COUNT(*) FROM msgs WHERE from_id=? OR to_id=?;",
+                    paramsv![contact_id as i32, contact_id as i32],
                 )
                 .await?
         } else {
@@ -899,7 +883,10 @@ impl Contact {
         if count_msgs == 0 {
             match context
                 .sql
-                .execute(sqlx::query("DELETE FROM contacts WHERE id=?;").bind(contact_id as i32))
+                .execute(
+                    "DELETE FROM contacts WHERE id=?;",
+                    paramsv![contact_id as i32],
+                )
                 .await
             {
                 Ok(_) => {
@@ -908,7 +895,7 @@ impl Contact {
                 }
                 Err(err) => {
                     error!(context, "delete_contact {} failed ({})", contact_id, err);
-                    return Err(err.into());
+                    return Err(err);
                 }
             }
         }
@@ -936,9 +923,8 @@ impl Contact {
         context
             .sql
             .execute(
-                sqlx::query("UPDATE contacts SET param=? WHERE id=?")
-                    .bind(self.param.to_string())
-                    .bind(self.id as i32),
+                "UPDATE contacts SET param=? WHERE id=?",
+                paramsv![self.param.to_string(), self.id as i32],
             )
             .await?;
         Ok(())
@@ -949,9 +935,8 @@ impl Contact {
         context
             .sql
             .execute(
-                sqlx::query("UPDATE contacts SET status=? WHERE id=?")
-                    .bind(&self.status)
-                    .bind(self.id as i32),
+                "UPDATE contacts SET status=? WHERE id=?",
+                paramsv![self.status, self.id as i32],
             )
             .await?;
         Ok(())
@@ -1093,18 +1078,14 @@ impl Contact {
         VerifiedStatus::Unverified
     }
 
-    pub async fn addr_equals_contact(
-        context: &Context,
-        addr: impl AsRef<str>,
-        contact_id: u32,
-    ) -> bool {
-        if addr.as_ref().is_empty() {
+    pub async fn addr_equals_contact(context: &Context, addr: &str, contact_id: u32) -> bool {
+        if addr.is_empty() {
             return false;
         }
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
             if !contact.addr.is_empty() {
-                let normalized_addr = addr_normalize(addr.as_ref());
+                let normalized_addr = addr_normalize(addr);
                 if contact.addr == normalized_addr {
                     return true;
                 }
@@ -1122,8 +1103,8 @@ impl Contact {
         let count = context
             .sql
             .count(
-                sqlx::query("SELECT COUNT(*) FROM contacts WHERE id>?;")
-                    .bind(DC_CONTACT_ID_LAST_SPECIAL),
+                "SELECT COUNT(*) FROM contacts WHERE id>?;",
+                paramsv![DC_CONTACT_ID_LAST_SPECIAL as i32],
             )
             .await?;
         Ok(count)
@@ -1136,7 +1117,10 @@ impl Contact {
 
         context
             .sql
-            .exists(sqlx::query("SELECT COUNT(*) FROM contacts WHERE id=?;").bind(contact_id))
+            .exists(
+                "SELECT COUNT(*) FROM contacts WHERE id=?;",
+                paramsv![contact_id as i32],
+            )
             .await
             .unwrap_or_default()
     }
@@ -1145,10 +1129,8 @@ impl Contact {
         context
             .sql
             .execute(
-                sqlx::query("UPDATE contacts SET origin=? WHERE id=? AND origin<?;")
-                    .bind(origin)
-                    .bind(contact_id)
-                    .bind(origin),
+                "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
+                paramsv![origin, contact_id as i32, origin],
             )
             .await
             .is_ok()
@@ -1172,23 +1154,23 @@ pub fn addr_normalize(addr: &str) -> &str {
     }
 }
 
-fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
+fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
     if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
-            if name.as_ref().is_empty() {
+            if name.is_empty() {
                 captures
                     .get(1)
                     .map_or("".to_string(), |m| normalize_name(m.as_str()))
             } else {
-                name.as_ref().to_string()
+                name.to_string()
             },
             captures
                 .get(2)
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (name.as_ref().to_string(), addr.as_ref().to_string())
+        (name.to_string(), addr.to_string())
     }
 }
 
@@ -1202,9 +1184,8 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             && context
                 .sql
                 .execute(
-                    sqlx::query("UPDATE contacts SET blocked=? WHERE id=?;")
-                        .bind(new_blocking as i32)
-                        .bind(contact_id),
+                    "UPDATE contacts SET blocked=? WHERE id=?;",
+                    paramsv![new_blocking as i32, contact_id as i32],
                 )
                 .await
                 .is_ok()
@@ -1217,18 +1198,14 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             if context
                 .sql
                 .execute(
-                    sqlx::query(
-                        r#"
+                    r#"
 UPDATE chats
 SET blocked=?
 WHERE type=? AND id IN (
   SELECT chat_id FROM chats_contacts WHERE contact_id=?
 );
 "#,
-                    )
-                    .bind(new_blocking)
-                    .bind(Chattype::Single)
-                    .bind(contact_id),
+                    paramsv![new_blocking, Chattype::Single, contact_id],
                 )
                 .await
                 .is_ok()
@@ -1299,13 +1276,31 @@ pub(crate) async fn set_profile_image(
 }
 
 /// Sets contact status.
-pub(crate) async fn set_status(context: &Context, contact_id: u32, status: String) -> Result<()> {
-    let mut contact = Contact::load_from_db(context, contact_id).await?;
+///
+/// For contact SELF, the status is not saved in the contact table, but as Config::Selfstatus.  This
+/// is only done if message is sent from Delta Chat and it is encrypted, to synchronize signature
+/// between Delta Chat devices.
+pub(crate) async fn set_status(
+    context: &Context,
+    contact_id: u32,
+    status: String,
+    encrypted: bool,
+    has_chat_version: bool,
+) -> Result<()> {
+    if contact_id == DC_CONTACT_ID_SELF {
+        if encrypted && has_chat_version {
+            context
+                .set_config(Config::Selfstatus, Some(&status))
+                .await?;
+        }
+    } else {
+        let mut contact = Contact::load_from_db(context, contact_id).await?;
 
-    if contact.status != status {
-        contact.status = status;
-        contact.update_status(context).await?;
-        context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+        if contact.status != status {
+            contact.status = status;
+            contact.update_status(context).await?;
+            context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+        }
     }
     Ok(())
 }
@@ -1332,13 +1327,13 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
 
 fn cat_fingerprint(
     ret: &mut String,
-    addr: impl AsRef<str>,
+    addr: &str,
     fingerprint_verified: impl AsRef<str>,
     fingerprint_unverified: impl AsRef<str>,
 ) {
     *ret += &format!(
         "\n\n{}:\n{}",
-        addr.as_ref(),
+        addr,
         if !fingerprint_verified.as_ref().is_empty() {
             fingerprint_verified.as_ref()
         } else {
@@ -1351,7 +1346,7 @@ fn cat_fingerprint(
     {
         *ret += &format!(
             "\n\n{} (alternative):\n{}",
-            addr.as_ref(),
+            addr,
             fingerprint_unverified.as_ref()
         );
     }
@@ -1396,6 +1391,7 @@ mod tests {
     use super::*;
 
     use crate::chat::send_text_msg;
+    use crate::message::Message;
     use crate::test_utils::TestContext;
 
     #[test]
@@ -1901,4 +1897,70 @@ CCCB 5AA9 F6E1 141C 9431
 
         Ok(())
     }
+
+    /// Tests that status is synchronized when sending encrypted BCC-self messages and not
+    /// synchronized when the message is not encrypted.
+    #[async_std::test]
+    async fn test_synchronize_status() -> Result<()> {
+        // Alice has two devices.
+        let alice1 = TestContext::new_alice().await;
+        let alice2 = TestContext::new_alice().await;
+
+        // Bob has one device.
+        let bob = TestContext::new_bob().await;
+
+        let default_status = alice1.get_config(Config::Selfstatus).await?;
+
+        alice1
+            .set_config(Config::Selfstatus, Some("New status"))
+            .await?;
+        let chat = alice1
+            .create_chat_with_contact("Bob", "bob@example.net")
+            .await;
+
+        // Alice sends a message to Bob from the first device.
+        send_text_msg(&alice1, chat.id, "Hello".to_string()).await?;
+        let sent_msg = alice1.pop_sent_msg().await;
+
+        // Message is not encrypted.
+        let message = Message::load_from_db(&alice1, sent_msg.sender_msg_id).await?;
+        assert!(!message.get_showpadlock());
+
+        // Alice's second devices receives a copy of outgoing message.
+        alice2.recv_msg(&sent_msg).await;
+
+        // Bob receives message.
+        bob.recv_msg(&sent_msg).await;
+
+        // Message was not encrypted, so status is not copied.
+        assert_eq!(alice2.get_config(Config::Selfstatus).await?, default_status);
+
+        // Bob replies.
+        let chat = bob
+            .create_chat_with_contact("Alice", "alice@example.com")
+            .await;
+
+        send_text_msg(&bob, chat.id, "Reply".to_string()).await?;
+        let sent_msg = bob.pop_sent_msg().await;
+        alice1.recv_msg(&sent_msg).await;
+        alice2.recv_msg(&sent_msg).await;
+
+        // Alice sends second message.
+        send_text_msg(&alice1, chat.id, "Hello".to_string()).await?;
+        let sent_msg = alice1.pop_sent_msg().await;
+
+        // Second message is encrypted.
+        let message = Message::load_from_db(&alice1, sent_msg.sender_msg_id).await?;
+        assert!(message.get_showpadlock());
+
+        // Alice's second devices receives a copy of second outgoing message.
+        alice2.recv_msg(&sent_msg).await;
+
+        assert_eq!(
+            alice2.get_config(Config::Selfstatus).await?,
+            Some("New status".to_string())
+        );
+
+        Ok(())
+    }
 }

src/context.rs

@@ -6,14 +6,12 @@ use std::ops::Deref;
 use std::time::{Instant, SystemTime};
 
 use anyhow::{bail, ensure, Result};
-use async_std::prelude::*;
 use async_std::{
     channel::{self, Receiver, Sender},
     path::{Path, PathBuf},
     sync::{Arc, Mutex, RwLock},
     task,
 };
-use sqlx::Row;
 
 use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
@@ -91,7 +89,7 @@ pub struct RunningState {
 pub fn get_info() -> BTreeMap<&'static str, String> {
     let mut res = BTreeMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
-    res.insert("sqlite_version", crate::sql::version().to_string());
+    res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("num_cpus", num_cpus::get().to_string());
     res.insert("level", "awesome".into());
@@ -290,7 +288,7 @@ impl Context {
             .unwrap_or_default();
         let journal_mode = self
             .sql
-            .query_get_value(sqlx::query("PRAGMA journal_mode;"))
+            .query_get_value("PRAGMA journal_mode;", paramsv![])
             .await?
             .unwrap_or_else(|| "unknown".to_string());
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
@@ -299,12 +297,12 @@ impl Context {
 
         let prv_key_cnt = self
             .sql
-            .count(sqlx::query("SELECT COUNT(*) FROM keypairs;"))
+            .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
             .await?;
 
         let pub_key_cnt = self
             .sql
-            .count(sqlx::query("SELECT COUNT(*) FROM acpeerstates;"))
+            .count("SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await?;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
             Ok(key) => key.fingerprint().hex(),
@@ -431,8 +429,8 @@ impl Context {
     pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
         let list = self
             .sql
-            .fetch(
-                sqlx::query(concat!(
+            .query_map(
+                concat!(
                     "SELECT m.id",
                     " FROM msgs m",
                     " LEFT JOIN contacts ct",
@@ -446,13 +444,17 @@ impl Context {
                     "   AND c.blocked=0",
                     "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
-                ))
-                .bind(MessageState::InFresh)
-                .bind(time()),
+                ),
+                paramsv![MessageState::InFresh, time()],
+                |row| row.get::<_, MsgId>(0),
+                |rows| {
+                    let mut list = Vec::new();
+                    for row in rows {
+                        list.push(row?);
+                    }
+                    Ok(list)
+                },
             )
-            .await?
-            .map(|row| row?.try_get("id"))
-            .collect::<sqlx::Result<_>>()
             .await?;
         Ok(list)
     }
@@ -461,49 +463,55 @@ impl Context {
     ///
     /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
     /// is `None` this searches messages from all chats.
-    pub async fn search_msgs(
-        &self,
-        chat_id: Option<ChatId>,
-        query: impl AsRef<str>,
-    ) -> Result<Vec<MsgId>> {
-        let real_query = query.as_ref().trim();
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: &str) -> Result<Vec<MsgId>> {
+        let real_query = query.trim();
         if real_query.is_empty() {
             return Ok(Vec::new());
         }
         let str_like_in_text = format!("%{}%", real_query);
-        let str_like_beg = format!("{}%", real_query);
+
+        let do_query = |query, params| {
+            self.sql.query_map(
+                query,
+                params,
+                |row| row.get::<_, MsgId>("id"),
+                |rows| {
+                    let mut ret = Vec::new();
+                    for id in rows {
+                        ret.push(id?);
+                    }
+                    Ok(ret)
+                },
+            )
+        };
 
         let list = if let Some(chat_id) = chat_id {
-            self.sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT m.id AS id, m.timestamp AS timestamp
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
                  WHERE m.chat_id=?
                    AND m.hidden=0
                    AND ct.blocked=0
-                   AND (txt LIKE ? OR ct.name LIKE ?)
+                   AND txt LIKE ?
                  ORDER BY m.timestamp,m.id;",
-                    )
-                    .bind(chat_id)
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
-                )
-                .await?
-                .map(|row| {
-                    let row = row?;
-                    let id = row.try_get::<MsgId, _>("id")?;
-                    Ok(id)
-                })
-                .collect::<sqlx::Result<Vec<MsgId>>>()
-                .await?
+                paramsv![chat_id, str_like_in_text],
+            )
+            .await?
         } else {
-            self.sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT m.id AS id, m.timestamp AS timestamp
+            // For performance reasons results are sorted only by `id`, that is in the order of
+            // message reception.
+            //
+            // Unlike chat view, sorting by `timestamp` is not necessary but slows down the query by
+            // ~25% according to benchmarks.
+            //
+            // To speed up incremental search, where queries for few characters usually return lots
+            // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
+            // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
+            // The limit is documented and UI may add a hint when getting 1000 results.
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -513,47 +521,37 @@ impl Context {
                    AND m.hidden=0
                    AND c.blocked=0
                    AND ct.blocked=0
-                   AND (m.txt LIKE ? OR ct.name LIKE ?)
-                 ORDER BY m.timestamp DESC,m.id DESC;",
-                    )
-                    .bind(str_like_in_text)
-                    .bind(str_like_beg),
-                )
-                .await?
-                .map(|row| {
-                    let row = row?;
-                    let id = row.try_get::<MsgId, _>("id")?;
-                    Ok(id)
-                })
-                .collect::<sqlx::Result<Vec<MsgId>>>()
-                .await?
+                   AND m.txt LIKE ?
+                 ORDER BY m.id DESC LIMIT 1000",
+                paramsv![str_like_in_text],
+            )
+            .await?
         };
 
         Ok(list)
     }
 
-    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_inbox(&self, folder_name: &str) -> Result<bool> {
         let inbox = self.get_config(Config::ConfiguredInboxFolder).await?;
-        Ok(inbox == Some(folder_name.as_ref().to_string()))
+        Ok(inbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_sentbox(&self, folder_name: &str) -> Result<bool> {
         let sentbox = self.get_config(Config::ConfiguredSentboxFolder).await?;
 
-        Ok(sentbox == Some(folder_name.as_ref().to_string()))
+        Ok(sentbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> Result<bool> {
+    pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
         let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
 
-        Ok(mvbox == Some(folder_name.as_ref().to_string()))
+        Ok(mvbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_spam_folder(&self, folder_name: impl AsRef<str>) -> Result<bool> {
-        let is_spam = self.get_config(Config::ConfiguredSpamFolder).await?
-            == Some(folder_name.as_ref().to_string());
+    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
+        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
 
-        Ok(is_spam)
+        Ok(spam.as_deref() == Some(folder_name))
     }
 
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
@@ -605,9 +603,13 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::chat::{get_chat_contacts, get_chat_msgs, set_muted, Chat, MuteDuration};
+    use crate::chat::{
+        get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
+    };
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::message::Message;
     use crate::test_utils::TestContext;
     use std::time::Duration;
     use strum::IntoEnumIterator;
@@ -735,9 +737,8 @@ mod tests {
         // we need to modify the database directly
         t.sql
             .execute(
-                sqlx::query("UPDATE chats SET muted_until=? WHERE id=?;")
-                    .bind(time() - 3600)
-                    .bind(bob.id),
+                "UPDATE chats SET muted_until=? WHERE id=?;",
+                paramsv![time() - 3600, bob.id],
             )
             .await
             .unwrap();
@@ -754,7 +755,10 @@ mod tests {
         // to test get_fresh_msgs() with invalid mute_until (everything < -1),
         // that results in "muted forever" by definition.
         t.sql
-            .execute(sqlx::query("UPDATE chats SET muted_until=-2 WHERE id=?;").bind(bob.id))
+            .execute(
+                "UPDATE chats SET muted_until=-2 WHERE id=?;",
+                paramsv![bob.id],
+            )
             .await
             .unwrap();
         let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
@@ -879,4 +883,93 @@ mod tests {
             }
         }
     }
+
+    #[async_std::test]
+    async fn test_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Global search finds nothing.
+        let res = alice.search_msgs(None, "foo").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds nothing.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert!(res.is_empty());
+
+        // Add messages to chat with Bob.
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.set_text(Some("foobar".to_string()));
+        send_msg(&alice, chat.id, &mut msg1).await?;
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_text(Some("barbaz".to_string()));
+        send_msg(&alice, chat.id, &mut msg2).await?;
+
+        // Global search with a part of text finds the message.
+        let res = alice.search_msgs(None, "ob").await?;
+        assert_eq!(res.len(), 1);
+
+        // Global search for "bar" matches both "foobar" and "barbaz".
+        let res = alice.search_msgs(None, "bar").await?;
+        assert_eq!(res.len(), 2);
+
+        // Message added later is returned first.
+        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.get(1), Some(&msg1.id));
+
+        // Global search with longer text does not find any message.
+        let res = alice.search_msgs(None, "foobarbaz").await?;
+        assert!(res.is_empty());
+
+        // Search for random string finds nothing.
+        let res = alice.search_msgs(None, "abc").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds the message.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1);
+
+        // Search in Saved Messages does not find the message.
+        let res = alice.search_msgs(Some(self_talk), "foo").await?;
+        assert!(res.is_empty());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_limit_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Add 999 messages
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foobar".to_string()));
+        for _ in 0..999 {
+            send_msg(&alice, chat.id, &mut msg).await?;
+        }
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 999);
+
+        // Add one more message, no limit yet
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // Add one more message, that one is truncated then
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // In-chat should not be not limited
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1001);
+
+        Ok(())
+    }
 }

src/dc_receive_imf.rs

@@ -1,16 +1,14 @@
 use std::convert::TryFrom;
 
 use anyhow::{bail, ensure, format_err, Result};
-use async_std::prelude::*;
 use itertools::join;
 use mailparse::SingleInfo;
 use num_traits::FromPrimitive;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use sha2::{Digest, Sha256};
-use sqlx::Row;
 
-use crate::chat::{self, Chat, ChatId, ProtectionStatus};
+use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::config::Config;
 use crate::constants::{
     Blocked, Chattype, ShowEmails, Viewtype, DC_CHAT_ID_TRASH, DC_CONTACT_ID_LAST_SPECIAL,
@@ -53,7 +51,7 @@ enum CreateEvent {
 pub async fn dc_receive_imf(
     context: &Context,
     imf_raw: &[u8],
-    server_folder: impl AsRef<str>,
+    server_folder: &str,
     server_uid: u32,
     seen: bool,
 ) -> Result<()> {
@@ -63,21 +61,14 @@ pub async fn dc_receive_imf(
 pub(crate) async fn dc_receive_imf_inner(
     context: &Context,
     imf_raw: &[u8],
-    server_folder: impl AsRef<str>,
+    server_folder: &str,
     server_uid: u32,
     seen: bool,
     fetching_existing_messages: bool,
 ) -> Result<()> {
     info!(
         context,
-        "Receiving message {}/{}, seen={}...",
-        if !server_folder.as_ref().is_empty() {
-            server_folder.as_ref()
-        } else {
-            "?"
-        },
-        server_uid,
-        seen
+        "Receiving message {}/{}, seen={}...", server_folder, server_uid, seen
     );
 
     if std::env::var(crate::DCC_MIME_DEBUG).unwrap_or_default() == "2" {
@@ -182,7 +173,7 @@ pub(crate) async fn dc_receive_imf_inner(
             imf_raw,
             incoming,
             incoming_origin,
-            server_folder.as_ref(),
+            server_folder,
             server_uid,
             &to_ids,
             &rfc724_mid,
@@ -249,6 +240,8 @@ pub(crate) async fn dc_receive_imf_inner(
             context,
             from_id,
             mime_parser.footer.clone().unwrap_or_default(),
+            mime_parser.was_encrypted(),
+            mime_parser.has_chat_version(),
         )
         .await
         {
@@ -262,12 +255,14 @@ pub(crate) async fn dc_receive_imf_inner(
     if !created_db_entries.is_empty() {
         if needs_delete_job || delete_server_after == Some(0) {
             for db_entry in &created_db_entries {
-                info!(context, "verbose (issue 2335): adding job after receive");
-                let mut params = Params::new();
-                params.set(Param::Arg, "comment: verbose (issue 2335) dc_receive_imf()");
                 job::add(
                     context,
-                    job::Job::new(Action::DeleteMsgOnImap, db_entry.1.to_u32(), params, 0),
+                    job::Job::new(
+                        Action::DeleteMsgOnImap,
+                        db_entry.1.to_u32(),
+                        Params::new(),
+                        0,
+                    ),
                 )
                 .await;
             }
@@ -368,7 +363,7 @@ async fn add_parts(
     imf_raw: &[u8],
     incoming: bool,
     incoming_origin: Origin,
-    server_folder: impl AsRef<str>,
+    server_folder: &str,
     server_uid: u32,
     to_ids: &ContactIds,
     rfc724_mid: &str,
@@ -396,9 +391,8 @@ async fn add_parts(
     if let Some((old_server_folder, old_server_uid, _)) =
         message::rfc724_mid_exists(context, rfc724_mid).await?
     {
-        if old_server_folder != server_folder.as_ref() || old_server_uid != server_uid {
-            message::update_server_uid(context, rfc724_mid, server_folder.as_ref(), server_uid)
-                .await;
+        if old_server_folder != server_folder || old_server_uid != server_uid {
+            message::update_server_uid(context, rfc724_mid, server_folder, server_uid).await;
         }
 
         warn!(context, "Message already in DB");
@@ -476,10 +470,9 @@ async fn add_parts(
             }
         }
 
-        let (test_normal_chat_id, test_normal_chat_id_blocked) =
-            chat::lookup_by_contact_id(context, from_id)
-                .await
-                .unwrap_or_default();
+        let test_normal_chat = ChatIdBlocked::lookup_by_contact(context, from_id)
+            .await
+            .unwrap_or_default();
 
         // get the chat_id - a chat_id here is no indicator that the chat is displayed in the normal list,
         // it might also be blocked and displayed in the deaddrop as a result
@@ -500,17 +493,18 @@ async fn add_parts(
         if chat_id.is_unset() {
             // try to create a group
 
-            let create_blocked =
-                if !test_normal_chat_id.is_unset() && test_normal_chat_id_blocked == Blocked::Not {
-                    Blocked::Not
-                } else {
-                    Blocked::Deaddrop
-                };
+            let create_blocked = match test_normal_chat {
+                Some(ChatIdBlocked {
+                    id: _,
+                    blocked: Blocked::Not,
+                }) => Blocked::Not,
+                _ => Blocked::Deaddrop,
+            };
 
             let (new_chat_id, new_chat_id_blocked) = create_or_lookup_group(
                 context,
                 &mut mime_parser,
-                if test_normal_chat_id.is_unset() {
+                if test_normal_chat.is_none() {
                     allow_creation
                 } else {
                     true
@@ -602,16 +596,17 @@ async fn add_parts(
                 Blocked::Deaddrop
             };
 
-            if !test_normal_chat_id.is_unset() {
-                *chat_id = test_normal_chat_id;
-                chat_id_blocked = test_normal_chat_id_blocked;
+            if let Some(chat) = test_normal_chat {
+                *chat_id = chat.id;
+                chat_id_blocked = chat.blocked;
             } else if allow_creation {
-                let (id, bl) =
-                    chat::create_or_lookup_by_contact_id(context, from_id, create_blocked)
-                        .await
-                        .unwrap_or_default();
-                *chat_id = id;
-                chat_id_blocked = bl;
+                if let Ok(chat) = ChatIdBlocked::get_for_contact(context, from_id, create_blocked)
+                    .await
+                    .log_err(context, "Failed to get (new) chat for contact")
+                {
+                    *chat_id = chat.id;
+                    chat_id_blocked = chat.blocked;
+                }
             }
             if !chat_id.is_unset() && Blocked::Not != chat_id_blocked {
                 if Blocked::Not == create_blocked {
@@ -656,7 +651,7 @@ async fn add_parts(
         let is_spam = (chat_id_blocked == Blocked::Deaddrop)
             && !incoming_origin.is_known()
             && (is_dc_message == MessengerMessage::No)
-            && context.is_spam_folder(&server_folder).await?;
+            && context.is_spam_folder(server_folder).await?;
         if is_spam {
             *chat_id = DC_CHAT_ID_TRASH;
             info!(context, "Message is probably spam (TRASH)");
@@ -690,7 +685,7 @@ async fn add_parts(
             }
         }
 
-        if !context.is_sentbox(&server_folder).await?
+        if !context.is_sentbox(server_folder).await?
             && mime_parser.get(HeaderDef::Received).is_none()
         {
             // Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them
@@ -727,11 +722,12 @@ async fn add_parts(
                 } else {
                     Blocked::Deaddrop
                 };
-                let (id, bl) = chat::create_or_lookup_by_contact_id(context, to_id, create_blocked)
-                    .await
-                    .unwrap_or_default();
-                *chat_id = id;
-                chat_id_blocked = bl;
+                if let Ok(chat) =
+                    ChatIdBlocked::get_for_contact(context, to_id, create_blocked).await
+                {
+                    *chat_id = chat.id;
+                    chat_id_blocked = chat.blocked;
+                }
 
                 if !chat_id.is_unset()
                     && Blocked::Not != chat_id_blocked
@@ -749,12 +745,14 @@ async fn add_parts(
         if chat_id.is_unset() && self_sent {
             // from_id==to_id==DC_CONTACT_ID_SELF - this is a self-sent messages,
             // maybe an Autocrypt Setup Message
-            let (id, bl) =
-                chat::create_or_lookup_by_contact_id(context, DC_CONTACT_ID_SELF, Blocked::Not)
+            if let Ok(chat) =
+                ChatIdBlocked::get_for_contact(context, DC_CONTACT_ID_SELF, Blocked::Not)
                     .await
-                    .unwrap_or_default();
-            *chat_id = id;
-            chat_id_blocked = bl;
+                    .log_err(context, "Failed to get (new) chat for contact")
+            {
+                *chat_id = chat.id;
+                chat_id_blocked = chat.blocked;
+            }
 
             if !chat_id.is_unset() && Blocked::Not != chat_id_blocked {
                 chat_id.unblock(context).await;
@@ -917,7 +915,7 @@ async fn add_parts(
 
     let subject = mime_parser.get_subject().unwrap_or_default();
 
-    let server_folder = server_folder.as_ref();
+    let mut parts = std::mem::replace(&mut mime_parser.parts, Vec::new());
     let is_system_message = mime_parser.is_system_message;
 
     // if indicated by the parser,
@@ -929,25 +927,59 @@ async fn add_parts(
 
     let mime_headers = if save_mime_headers || save_mime_modified {
         if mime_parser.was_encrypted() && !mime_parser.decoded_data.is_empty() {
-            String::from_utf8_lossy(&mime_parser.decoded_data)
+            String::from_utf8_lossy(&mime_parser.decoded_data).to_string()
         } else {
-            String::from_utf8_lossy(imf_raw)
+            String::from_utf8_lossy(imf_raw).to_string()
         }
     } else {
         "".into()
     };
 
-    for part in &mut mime_parser.parts {
+    let sent_timestamp = *sent_timestamp;
+    let is_hidden = *hidden;
+    let chat_id = *chat_id;
+
+    // TODO: can this clone be avoided?
+    let rfc724_mid = rfc724_mid.to_string();
+
+    let mut is_hidden = is_hidden;
+    let mut ids = Vec::with_capacity(parts.len());
+
+    let conn = context.sql.get_conn().await?;
+
+    for part in &mut parts {
         let mut txt_raw = "".to_string();
+        let mut stmt = conn.prepare_cached(
+            r#"
+INSERT INTO msgs
+  (
+    rfc724_mid, server_folder, server_uid, chat_id, 
+    from_id, to_id, timestamp, timestamp_sent, 
+    timestamp_rcvd, type, state, msgrmsg, 
+    txt, subject, txt_raw, param, 
+    bytes, hidden, mime_headers, mime_in_reply_to,
+    mime_references, mime_modified, error, ephemeral_timer,
+    ephemeral_timestamp
+  )
+  VALUES (
+    ?, ?, ?, ?,
+    ?, ?, ?, ?,
+    ?, ?, ?, ?,
+    ?, ?, ?, ?,
+    ?, ?, ?, ?,
+    ?, ?, ?, ?,
+    ?
+  );
+"#,
+        )?;
 
         let is_location_kml =
             location_kml_is && icnt == 1 && (part.msg == "-location-" || part.msg.is_empty());
 
         if is_mdn || is_location_kml {
-            *hidden = true;
+            is_hidden = true;
             if incoming {
-                // Set the state to InSeen so that precheck_imf() adds a markseen job after we moved the message
-                state = MessageState::InSeen;
+                state = MessageState::InSeen; // Set the state to InSeen so that precheck_imf() adds a markseen job after we moved the message
             }
         }
 
@@ -978,83 +1010,61 @@ async fn add_parts(
         // also change `MsgId::trash()` and `delete_expired_messages()`
         let trash = chat_id.is_trash();
 
-        context
-            .sql
-            .execute(
-                sqlx::query(
-                    r#"
-INSERT INTO msgs
-  (
-    rfc724_mid, server_folder, server_uid, chat_id, 
-    from_id, to_id, timestamp, timestamp_sent, 
-    timestamp_rcvd, type, state, msgrmsg, 
-    txt, subject, txt_raw, param, 
-    bytes, hidden, mime_headers, mime_in_reply_to,
-    mime_references, mime_modified, error, ephemeral_timer,
-    ephemeral_timestamp
-  )
-  VALUES (
-    ?, ?, ?, ?,
-    ?, ?, ?, ?,
-    ?, ?, ?, ?,
-    ?, ?, ?, ?,
-    ?, ?, ?, ?,
-    ?, ?, ?, ?,
-    ?
-  );
-"#,
-                )
-                .bind(rfc724_mid)
-                .bind(server_folder)
-                .bind(server_uid as i32)
-                .bind(*chat_id)
-                .bind(if trash { 0 } else { from_id as i32 })
-                .bind(if trash { 0 } else { to_id as i32 })
-                .bind(sort_timestamp)
-                .bind(*sent_timestamp)
-                .bind(rcvd_timestamp)
-                .bind(part.typ)
-                .bind(state)
-                .bind(is_dc_message)
-                .bind(if trash { "" } else { &part.msg })
-                .bind(if trash { "" } else { &subject })
-                // txt_raw might contain invalid utf8
-                .bind(if trash { "" } else { &txt_raw })
-                .bind(if trash {
-                    "".to_string()
-                } else {
-                    part.param.to_string()
-                })
-                .bind(part.bytes as i64)
-                .bind(*hidden)
-                .bind(if (save_mime_headers || mime_modified) && !trash {
-                    mime_headers.to_string()
-                } else {
-                    "".to_string()
-                })
-                .bind(&mime_in_reply_to)
-                .bind(&mime_references)
-                .bind(&mime_modified)
-                .bind(part.error.take().unwrap_or_default())
-                .bind(ephemeral_timer)
-                .bind(ephemeral_timestamp),
-            )
-            .await?;
-        let msg_id = MsgId::new(u32::try_from(
-            context
-                .sql
-                .get_rowid("msgs", "rfc724_mid", &rfc724_mid)
-                .await?,
-        )?);
+        stmt.execute(paramsv![
+            rfc724_mid,
+            server_folder,
+            server_uid as i32,
+            chat_id,
+            if trash { 0 } else { from_id as i32 },
+            if trash { 0 } else { to_id as i32 },
+            sort_timestamp,
+            sent_timestamp,
+            rcvd_timestamp,
+            part.typ,
+            state,
+            is_dc_message,
+            if trash { "" } else { &part.msg },
+            if trash { "" } else { &subject },
+            // txt_raw might contain invalid utf8
+            if trash { "" } else { &txt_raw },
+            if trash {
+                "".to_string()
+            } else {
+                part.param.to_string()
+            },
+            part.bytes as isize,
+            is_hidden,
+            if (save_mime_headers || mime_modified) && !trash {
+                mime_headers.to_string()
+            } else {
+                "".to_string()
+            },
+            mime_in_reply_to,
+            mime_references,
+            mime_modified,
+            part.error.take().unwrap_or_default(),
+            ephemeral_timer,
+            ephemeral_timestamp
+        ])?;
+        let row_id = conn.last_insert_rowid();
 
-        created_db_entries.push((*chat_id, msg_id));
-        *insert_msg_id = msg_id;
+        drop(stmt);
+        ids.push(MsgId::new(u32::try_from(row_id)?));
+    }
+    drop(conn);
+
+    if let Some(id) = ids.iter().last() {
+        *insert_msg_id = *id;
     }
 
-    if !*hidden {
+    if !is_hidden {
         chat_id.unarchive(context).await?;
     }
 
+    *hidden = is_hidden;
+    created_db_entries.extend(ids.iter().map(|id| (chat_id, *id)));
+    mime_parser.parts = parts;
+
     info!(
         context,
         "Message has {} parts and is assigned to chat #{}.", icnt, chat_id,
@@ -1062,7 +1072,7 @@ INSERT INTO msgs
 
     // new outgoing message from another device marks the chat as noticed.
     if !incoming && !*hidden && !chat_id.is_special() {
-        chat::marknoticed_chat_if_older_than(context, *chat_id, sort_timestamp).await?;
+        chat::marknoticed_chat_if_older_than(context, chat_id, sort_timestamp).await?;
     }
 
     // check event to send
@@ -1092,7 +1102,7 @@ INSERT INTO msgs
         Ok(())
     }
     if !is_mdn {
-        update_last_subject(context, *chat_id, mime_parser)
+        update_last_subject(context, chat_id, mime_parser)
             .await
             .ok_or_log_msg(context, "Could not update LastSubject of chat");
     }
@@ -1174,9 +1184,8 @@ async fn calc_sort_timestamp(
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
-                sqlx::query("SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?")
-                    .bind(chat_id)
-                    .bind(MessageState::InFresh),
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
+                paramsv![chat_id, MessageState::InFresh],
             )
             .await?;
 
@@ -1488,9 +1497,8 @@ async fn create_or_lookup_group(
                 if context
                     .sql
                     .execute(
-                        sqlx::query("UPDATE chats SET name=? WHERE id=?;")
-                            .bind(grpname.to_string())
-                            .bind(chat_id),
+                        "UPDATE chats SET name=? WHERE id=?;",
+                        paramsv![grpname.to_string(), chat_id],
                     )
                     .await
                     .is_ok()
@@ -1527,7 +1535,10 @@ async fn create_or_lookup_group(
             // start from scratch.
             context
                 .sql
-                .execute(sqlx::query("DELETE FROM chats_contacts WHERE chat_id=?;").bind(chat_id))
+                .execute(
+                    "DELETE FROM chats_contacts WHERE chat_id=?;",
+                    paramsv![chat_id],
+                )
                 .await
                 .ok();
 
@@ -1769,22 +1780,19 @@ async fn create_multiuser_record(
     create_blocked: Blocked,
     create_protected: ProtectionStatus,
 ) -> Result<ChatId> {
-    context.sql.execute(
-        sqlx::query(
-            "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected) VALUES(?, ?, ?, ?, ?, ?);")
-            .bind(chattype)
-            .bind(grpname.as_ref())
-            .bind(grpid.as_ref())
-            .bind(create_blocked)
-            .bind(time())
-            .bind(create_protected)
+    let row_id =
+    context.sql.insert(
+        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp, protected) VALUES(?, ?, ?, ?, ?, ?);",
+        paramsv![
+            chattype,
+            grpname.as_ref(),
+            grpid.as_ref(),
+            create_blocked,
+            time(),
+            create_protected,
+        ],
     ).await?;
 
-    let row_id = context
-        .sql
-        .get_rowid("chats", "grpid", grpid.as_ref())
-        .await?;
-
     let chat_id = ChatId::new(u32::try_from(row_id)?);
     info!(
         context,
@@ -1816,31 +1824,34 @@ async fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> Result<St
         .unwrap_or_else(|| "no-self".to_string())
         .to_lowercase();
 
-    let q = format!(
-        "SELECT addr FROM contacts WHERE id IN({}) AND id!=1", // 1=DC_CONTACT_ID_SELF
-        member_ids_str
-    );
-
-    let mut members = member_cs;
-
-    if let Ok(rows) = context.sql.fetch(sqlx::query(&q)).await {
-        let mut addrs = rows
-            .map(|row| row?.try_get::<String, _>(0))
-            .collect::<sqlx::Result<Vec<_>>>()
-            .await?;
-        addrs.sort();
-        for addr in &addrs {
-            members += ",";
-            members += &addr.to_lowercase();
-        }
-    }
+    let members = context
+        .sql
+        .query_map(
+            format!(
+                "SELECT addr FROM contacts WHERE id IN({}) AND id!=1", // 1=DC_CONTACT_ID_SELF
+                member_ids_str
+            ),
+            paramsv![],
+            |row| row.get::<_, String>(0),
+            |rows| {
+                let mut addrs = rows.collect::<std::result::Result<Vec<_>, _>>()?;
+                addrs.sort();
+                let mut acc = member_cs.clone();
+                for addr in &addrs {
+                    acc += ",";
+                    acc += &addr.to_lowercase();
+                }
+                Ok(acc)
+            },
+        )
+        .await?;
 
     Ok(hex_hash(&members))
 }
 
 #[allow(clippy::indexing_slicing)]
-fn hex_hash(s: impl AsRef<str>) -> String {
-    let bytes = s.as_ref().as_bytes();
+fn hex_hash(s: &str) -> String {
+    let bytes = s.as_bytes();
     let result = Sha256::digest(bytes);
     hex::encode(&result[..8])
 }
@@ -1900,26 +1911,34 @@ async fn check_verified_properties(
     }
     let to_ids_str = join(to_ids.iter().map(|x| x.to_string()), ",");
 
-    let q = format!(
-        "SELECT c.addr, LENGTH(ps.verified_key_fingerprint)  FROM contacts c  \
+    let rows = context
+        .sql
+        .query_map(
+            format!(
+                "SELECT c.addr, LENGTH(ps.verified_key_fingerprint)  FROM contacts c  \
              LEFT JOIN acpeerstates ps ON c.addr=ps.addr  WHERE c.id IN({}) ",
-        to_ids_str
-    );
-
-    let mut rows = context.sql.fetch(sqlx::query(&q)).await?;
-
-    while let Some(row) = rows.next().await {
-        let row = row?;
-        let to_addr: String = row.try_get(0)?;
-        let mut is_verified = row.try_get::<i32, _>(1)? != 0;
+                to_ids_str
+            ),
+            paramsv![],
+            |row| {
+                let to_addr: String = row.get(0)?;
+                let is_verified: i32 = row.get(1).unwrap_or(0);
+                Ok((to_addr, is_verified != 0))
+            },
+            |rows| {
+                rows.collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?;
 
+    for (to_addr, mut is_verified) in rows.into_iter() {
         info!(
             context,
             "check_verified_properties: {:?} self={:?}",
             to_addr,
             context.is_self_addr(&to_addr).await
         );
-
         let peerstate = Peerstate::from_addr(context, &to_addr).await?;
 
         // mark gossiped keys (if any) as verified
@@ -2065,7 +2084,7 @@ async fn add_or_lookup_contact_by_addr(
     let display_name_normalized = display_name.map(normalize_name).unwrap_or_default();
 
     let (row_id, _modified) =
-        Contact::add_or_lookup(context, display_name_normalized, addr, origin).await?;
+        Contact::add_or_lookup(context, &display_name_normalized, addr, origin).await?;
     ensure!(row_id > 0, "could not add contact: {:?}", addr);
 
     Ok(row_id)
@@ -2334,7 +2353,7 @@ mod tests {
         let t = TestContext::new_alice().await;
 
         let bob_id = Contact::create(&t, "bob", "bob@example.com").await.unwrap();
-        let one2one_id = chat::create_by_contact_id(&t, bob_id).await.unwrap();
+        let one2one_id = ChatId::create_for_contact(&t, bob_id).await.unwrap();
         one2one_id
             .set_visibility(&t, ChatVisibility::Archived)
             .await
@@ -2503,7 +2522,7 @@ mod tests {
         let contact_id = Contact::create(&t, "foobar", "foobar@example.com")
             .await
             .unwrap();
-        let chat_id = chat::create_by_contact_id(&t, contact_id).await.unwrap();
+        let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap();
         dc_receive_imf(
             &t,
             b"From: =?UTF-8?B?0JjQvNGPLCDQpNCw0LzQuNC70LjRjw==?= <foobar@example.com>\n\
@@ -3967,4 +3986,33 @@ YEAAAAAA!.
             .await,
         );
     }
+
+    #[async_std::test]
+    async fn test_dont_show_all_outgoing_msgs_in_self_chat() {
+        // Regression test for https://github.com/deltachat/deltachat-android/issues/1940:
+        // Some servers add a `Bcc: <Self>` header, which caused all outgoing messages to
+        // be shown in the self-chat.
+        let t = TestContext::new_alice().await;
+
+        dc_receive_imf(
+            &t,
+            b"Bcc: alice@example.com
+Received: from [127.0.0.1]
+Subject: s
+Chat-Version: 1.0
+Message-ID: <abcd@gmail.com>
+To: <me@other.maildomain.com>
+From: <alice@example.com>
+
+Message content",
+            "Inbox",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+
+        let msg = t.get_last_msg().await;
+        assert_ne!(msg.chat_id, t.get_self_chat().await.id);
+    }
 }

src/dc_tools.rs

@@ -632,10 +632,17 @@ impl FromStr for EmailAddress {
     }
 }
 
+impl rusqlite::types::ToSql for EmailAddress {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Text(self.to_string());
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
-pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .as_ref()
         .replace("\n", " ")
         .replace("\r", " ")
         .trim()
@@ -879,16 +886,6 @@ mod tests {
         }
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
-        if dc_file_exist!(context, "$BLOBDIR/foobar").await
-            || dc_file_exist!(context, "$BLOBDIR/dada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar").await;
-            dc_delete_file(context, "$BLOBDIR/dada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
-        }
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());

src/dehtml.rs

@@ -314,6 +314,7 @@ mod tests {
                 "[ Foo ](https://example.com)",
             ),
             ("<b> bar </b>", "* bar *"),
+            ("<i>foo</i>", "_foo_"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
             // Despite missing ', this should be shown:
@@ -391,6 +392,13 @@ mod tests {
         assert_eq!(txt.trim(), "lots of text");
     }
 
+    #[test]
+    fn test_pre_tag() {
+        let input = "<html><pre>\ntwo\nlines\n</pre></html>";
+        let txt = dehtml(input).unwrap();
+        assert_eq!(txt.trim(), "two\nlines");
+    }
+
     #[async_std::test]
     async fn test_quote_div() {
         let input = include_str!("../test-data/message/gmx-quote-body.eml");

src/ephemeral.rs

@@ -61,21 +61,20 @@ use std::num::ParseIntError;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
-use anyhow::{ensure, Context as _, Error};
+use anyhow::{ensure, Context as _, Result};
 use async_std::task;
 use serde::{Deserialize, Serialize};
-use sqlx::Row;
 
-use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::chat::{send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::events::EventType;
+use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::sql;
 use crate::stock_str;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
@@ -121,51 +120,39 @@ impl FromStr for Timer {
     }
 }
 
-impl sqlx::Type<sqlx::Sqlite> for Timer {
-    fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
-        <i64 as sqlx::Type<_>>::type_info()
-    }
-
-    fn compatible(ty: &sqlx::sqlite::SqliteTypeInfo) -> bool {
-        <i64 as sqlx::Type<_>>::compatible(ty)
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
     }
 }
 
-impl<'q> sqlx::Encode<'q, sqlx::Sqlite> for Timer {
-    fn encode_by_ref(
-        &self,
-        args: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>,
-    ) -> sqlx::encode::IsNull {
-        args.push(sqlx::sqlite::SqliteArgumentValue::Int64(
-            self.to_u32() as i64
-        ));
-
-        sqlx::encode::IsNull::No
-    }
-}
-
-impl<'r> sqlx::Decode<'r, sqlx::Sqlite> for Timer {
-    fn decode(value: sqlx::sqlite::SqliteValueRef<'r>) -> Result<Self, sqlx::error::BoxDynError> {
-        let value: i64 = sqlx::Decode::decode(value)?;
-        if value == 0 {
-            Ok(Self::Disabled)
-        } else if let Ok(duration) = u32::try_from(value) {
-            Ok(Self::Enabled { duration })
-        } else {
-            Err(Box::new(sqlx::Error::Decode(Box::new(
-                crate::error::OutOfRangeError,
-            ))))
-        }
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
     }
 }
 
 impl ChatId {
     /// Get ephemeral message timer value in seconds.
-    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer> {
         let timer = context
             .sql
             .query_get_value(
-                sqlx::query("SELECT ephemeral_timer FROM chats WHERE id=?;").bind(self),
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
             )
             .await?;
         Ok(timer.unwrap_or_default())
@@ -179,19 +166,16 @@ impl ChatId {
         self,
         context: &Context,
         timer: Timer,
-    ) -> Result<(), Error> {
+    ) -> Result<()> {
         ensure!(!self.is_special(), "Invalid chat ID");
 
         context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE chats
+                "UPDATE chats
              SET ephemeral_timer=?
              WHERE id=?;",
-                )
-                .bind(timer)
-                .bind(self),
+                paramsv![timer, self],
             )
             .await?;
 
@@ -205,7 +189,7 @@ impl ChatId {
     /// Set ephemeral message timer value in seconds.
     ///
     /// If timer value is 0, disable ephemeral message timer.
-    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<()> {
         if timer == self.get_ephemeral_timer(context).await? {
             return Ok(());
         }
@@ -230,45 +214,44 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     from_id: u32,
 ) -> String {
     match timer {
-        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id as u32).await,
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
             0..=59 => {
-                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id as u32)
-                    .await
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
             }
-            60 => stock_str::msg_ephemeral_timer_minute(context, from_id as u32).await,
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
             61..=3599 => {
                 stock_str::msg_ephemeral_timer_minutes(
                     context,
                     format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id as u32).await,
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
             3601..=86399 => {
                 stock_str::msg_ephemeral_timer_hours(
                     context,
                     format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            86400 => stock_str::msg_ephemeral_timer_day(context, from_id as u32).await,
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
             86401..=604_799 => {
                 stock_str::msg_ephemeral_timer_days(
                     context,
                     format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
-            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id as u32).await,
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
             _ => {
                 stock_str::msg_ephemeral_timer_weeks(
                     context,
                     format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
-                    from_id as u32,
+                    from_id,
                 )
                 .await
             }
@@ -281,15 +264,14 @@ impl MsgId {
     pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
         let res = match context
             .sql
-            .query_get_value::<_, i64>(
-                sqlx::query("SELECT ephemeral_timer FROM msgs WHERE id=?").bind(self),
+            .query_get_value(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
             )
             .await?
         {
             None | Some(0) => Timer::Disabled,
-            Some(duration) => Timer::Enabled {
-                duration: u32::try_from(duration)?,
-            },
+            Some(duration) => Timer::Enabled { duration },
         };
         Ok(res)
     }
@@ -302,14 +284,10 @@ impl MsgId {
             context
                 .sql
                 .execute(
-                    sqlx::query(
-                        "UPDATE msgs SET ephemeral_timestamp = ? \
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
                 WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
                 AND id = ?",
-                    )
-                    .bind(ephemeral_timestamp)
-                    .bind(ephemeral_timestamp)
-                    .bind(self),
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
                 )
                 .await?;
             schedule_ephemeral_task(context).await;
@@ -326,14 +304,13 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
     let mut updated = context
         .sql
         .execute(
-            sqlx::query(
-                // If you change which information is removed here, also change MsgId::trash() and
-                // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-                r#"
+            // If you change which information is removed here, also change MsgId::trash() and
+            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
+            r#"
 UPDATE msgs
 SET 
   chat_id=?, txt='', subject='', txt_raw='', 
@@ -343,24 +320,19 @@ WHERE
   AND ephemeral_timestamp <= ?
   AND chat_id != ?
 "#,
-            )
-            .bind(DC_CHAT_ID_TRASH)
-            .bind(time())
-            .bind(DC_CHAT_ID_TRASH),
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
         .await
         .context("update failed")?
         > 0;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
+        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            .await?
+            .unwrap_or_default();
 
         let threshold_timestamp = time() - delete_device_after;
 
@@ -371,19 +343,19 @@ WHERE
         let rows_modified = context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE msgs \
+                "UPDATE msgs \
              SET txt = 'DELETED', chat_id = ? \
              WHERE timestamp < ? \
              AND chat_id > ? \
              AND chat_id != ? \
              AND chat_id != ?",
-                )
-                .bind(DC_CHAT_ID_TRASH)
-                .bind(threshold_timestamp)
-                .bind(DC_CHAT_ID_LAST_SPECIAL)
-                .bind(self_chat_id)
-                .bind(device_chat_id),
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
             )
             .await
             .context("deleted update failed")?;
@@ -409,8 +381,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
         .query_get_value(
-            sqlx::query(
-                r#"
+            r#"
     SELECT ephemeral_timestamp
     FROM msgs
     WHERE ephemeral_timestamp != 0
@@ -418,8 +389,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
     ORDER BY ephemeral_timestamp ASC
     LIMIT 1;
     "#,
-            )
-            .bind(DC_CHAT_ID_TRASH), // Trash contains already deleted messages, skip them
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
     {
@@ -472,7 +442,7 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 ///
 /// It looks up the trash chat too, to find messages that are already
 /// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
     let threshold_timestamp = match context.get_config_delete_server_after().await? {
@@ -480,29 +450,24 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
         Some(delete_server_after) => now - delete_server_after,
     };
 
-    let row = context
+    context
         .sql
-        .fetch_optional(
-            sqlx::query(
-                "SELECT id FROM msgs \
+        .query_row_optional(
+            "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
+         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            )
-            .bind(threshold_timestamp)
-            .bind(now),
+            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                Ok(msg_id)
+            },
         )
-        .await?;
-
-    if let Some(row) = row {
-        let msg_id = row.try_get(0)?;
-        Ok(Some(msg_id))
-    } else {
-        Ok(None)
-    }
+        .await
 }
 
 /// Start ephemeral timers for seen messages if they are not started
@@ -514,21 +479,21 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
 ///
 /// This function is supposed to be called in the background,
 /// e.g. from housekeeping task.
-pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
     context
         .sql
         .execute(
-            sqlx::query(
-                "UPDATE msgs \
+            "UPDATE msgs \
     SET ephemeral_timestamp = ? + ephemeral_timer \
     WHERE ephemeral_timer > 0 \
     AND ephemeral_timestamp = 0 \
     AND state NOT IN (?, ?, ?)",
-            )
-            .bind(time())
-            .bind(MessageState::InFresh)
-            .bind(MessageState::InNoticed)
-            .bind(MessageState::OutDraft),
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
         )
         .await?;
 
@@ -537,6 +502,7 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use crate::param::Params;
     use async_std::task::sleep;
 
     use super::*;
@@ -758,7 +724,34 @@ mod tests {
 
         sleep(Duration::from_millis(1100)).await;
 
+        // Check checks that the msg was deleted locally
         check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+
+        // Check that the msg will be deleted on the server
+        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
+        t.sql
+            .execute(
+                "UPDATE msgs SET server_uid=1 WHERE id=?",
+                paramsv![msg.sender_msg_id],
+            )
+            .await
+            .unwrap();
+        let job = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(
+            job,
+            Some(job::Job::new(
+                job::Action::DeleteMsgOnImap,
+                msg.sender_msg_id.to_u32(),
+                Params::new(),
+                0,
+            ))
+        );
+        // Let's assume that executing the job fails on first try and the job is saved to the db
+        job.unwrap().save(&t).await.unwrap();
+
+        // Make sure that we don't get yet another job when loading from db
+        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(job2, None);
     }
 
     async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
@@ -778,7 +771,7 @@ mod tests {
             assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
             let rawtxt: Option<String> = t
                 .sql
-                .query_get_value(sqlx::query("SELECT txt_raw FROM msgs WHERE id=?;").bind(msg_id))
+                .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
                 .await
                 .unwrap();
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);

src/error.rs

@@ -1,9 +1,5 @@
 //! # Error handling
 
-#[derive(Debug, thiserror::Error)]
-#[error("Out of Range")]
-pub struct OutOfRangeError;
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -213,6 +213,8 @@ pub enum EventType {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
+    ///
+    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 

src/export_chat.rs

@@ -0,0 +1,339 @@
+//! Export chats module
+//!
+//! ## Export Format
+//! The format of an exported chat is a zip file with the following structure:
+//! ```text
+//! ├── blobs/          # all files that are referenced by the chat
+//! ├── msg_info/
+//! │   └── [msg_id].txt # message info
+//! ├── msg_source/
+//! │   └── [msg_id].eml # email sourcecode of messages if availible¹
+//! └── chat.json        # chat info, messages and message authors
+//! ```
+//! ##### ¹ Saving Mime header
+//! To save the mime header you need to have the config option [`SaveMimeHeaders`] enabled.
+//! This option saves the mime headers on future messages. Normaly the original email source code is discarded to save space.
+//! You can use the repl tool to do this job:
+//! ```sh
+//! $ cargo run --example repl --features=repl /path/to/account/db.sqlite
+//! > set save_mime_headers 1
+//! ```
+//! [`SaveMimeHeaders`]: ../config/enum.Config.html#variant.SaveMimeHeaders
+
+use crate::chat::*;
+use crate::constants::Viewtype;
+use crate::constants::DC_GCM_ADDDAYMARKER;
+use crate::contact::*;
+use crate::context::Context;
+// use crate::error::Error;
+use crate::dc_tools::time;
+use crate::message::*;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::prelude::*;
+use std::path::Path;
+use zip::write::FileOptions;
+
+use crate::location::Location;
+use serde::Serialize;
+
+#[derive(Debug)]
+struct ExportChatResult {
+    chat_json: String,
+    // locations_geo_json: String,
+    message_ids: Vec<MsgId>,
+    referenced_blobs: Vec<String>,
+}
+
+pub async fn export_chat_to_zip(context: &Context, chat_id: ChatId, filename: &str) {
+    let res = export_chat_data(&context, chat_id).await;
+    let destination = std::path::Path::new(filename);
+    let pack_res = pack_exported_chat(&context, res, destination).await;
+    match &pack_res {
+        Ok(()) => println!("Exported chat successfully to {}", filename),
+        Err(err) => println!("Error {:?}", err),
+    };
+}
+
+async fn pack_exported_chat(
+    context: &Context,
+    artifact: ExportChatResult,
+    destination: &Path,
+) -> zip::result::ZipResult<()> {
+    let file = std::fs::File::create(&destination).unwrap();
+
+    let mut zip = zip::ZipWriter::new(file);
+
+    zip.start_file("chat.json", Default::default())?;
+    zip.write_all(artifact.chat_json.as_bytes())?;
+
+    zip.add_directory("blobs/", Default::default())?;
+
+    let options = FileOptions::default();
+    for blob_name in artifact.referenced_blobs {
+        let path = context.get_blobdir().join(&blob_name);
+
+        // println!("adding file {:?} as {:?} ...", path, &blob_name);
+        zip.start_file(format!("blobs/{}", &blob_name), options)?;
+        let mut f = File::open(path)?;
+
+        let mut buffer = Vec::new();
+        f.read_to_end(&mut buffer)?;
+        zip.write_all(&*buffer)?;
+        buffer.clear();
+    }
+
+    zip.add_directory("msg_info/", Default::default())?;
+    zip.add_directory("msg_source/", Default::default())?;
+    for id in artifact.message_ids {
+        zip.start_file(format!("msg_info/{}.txt", id.to_u32()), options)?;
+        zip.write_all((get_msg_info(&context, id).await).as_bytes())?;
+        if let Some(mime_headers) = get_mime_headers(&context, id).await {
+            zip.start_file(format!("msg_source/{}.eml", id.to_u32()), options)?;
+            zip.write_all((mime_headers).as_bytes())?;
+        }
+    }
+
+    zip.finish()?;
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct ChatJSON {
+    chat_json_version: u8,
+    export_timestamp: i64,
+    name: String,
+    color: String,
+    profile_img: Option<String>,
+    contacts: HashMap<u32, ContactJSON>,
+    referenced_external_messages:Vec<ChatItemJSON>,
+    messages: Vec<ChatItemJSON>,
+    locations: Vec<Location>,
+}
+
+#[derive(Serialize)]
+struct ContactJSON {
+    name: String,
+    email: String,
+    color: String,
+    profile_img: Option<String>,
+}
+
+#[derive(Serialize)]
+struct FileReference {
+    name: String,
+    filesize: u64,
+    mime: String,
+    path: String,
+}
+
+#[derive(Serialize)]
+struct Qoute {
+    quoted_text: String,
+    message_id: Option<u32>,
+}
+
+#[derive(Serialize)]
+#[serde(tag = "type")]
+enum ChatItemJSON {
+    Message {
+        id: u32,
+        author_id: u32, // from_id
+        view_type: Viewtype,
+        timestamp_sort: i64,
+        timestamp_sent: i64,
+        timestamp_rcvd: i64,
+        text: Option<String>,
+        attachment: Option<FileReference>,
+        location_id: Option<u32>,
+        is_info_message: bool,
+        show_padlock: bool,
+        state: MessageState,
+        is_forwarded: bool,
+        quote: Option<Qoute>
+    },
+    MessageError {
+        id: u32,
+        error: String,
+    },
+    DayMarker {
+        timestamp: i64,
+    },
+}
+
+impl ChatItemJSON {
+    pub async fn from_message(message: &Message, context: &Context) -> ChatItemJSON {
+        let msg_id = message.get_id();
+        ChatItemJSON::Message {
+            id: msg_id.to_u32(),
+            author_id: message.get_from_id(), // from_id
+            view_type: message.get_viewtype(),
+            timestamp_sort: message.timestamp_sort,
+            timestamp_sent: message.timestamp_sent,
+            timestamp_rcvd: message.timestamp_rcvd,
+            text: message.get_text(),
+            attachment: match message.get_file(context) {
+                Some(file) => Some(FileReference {
+                    name: message.get_filename().unwrap_or_else(|| "".to_owned()),
+                    filesize: message.get_filebytes(context).await,
+                    mime: message.get_filemime().unwrap_or_else(|| "".to_owned()),
+                    path: format!(
+                        "blobs/{}",
+                        file.file_name()
+                            .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                            .to_str()
+                            .unwrap()
+                    ),
+                }),
+                None => None,
+            },
+            location_id: match message.has_location() {
+                true => Some(message.location_id),
+                false => None,
+            },
+            is_info_message: message.is_info(),
+            show_padlock: message.get_showpadlock(),
+            state: message.get_state(),
+            is_forwarded: message.is_forwarded(),
+            quote: match message.quoted_text() {
+                Some(text) => match message.quoted_message(&context).await {
+                    Ok(Some(msg)) => Some(Qoute {
+                        quoted_text: text,
+                        message_id: Some(msg.get_id().to_u32())
+                    }),
+                    Err(_) | Ok(None) => Some(Qoute {
+                        quoted_text: text,
+                        message_id: None
+                    })
+                }
+                None => None
+            }
+        }
+    }
+}
+
+async fn export_chat_data(context: &Context, chat_id: ChatId) -> ExportChatResult {
+    let mut blobs = Vec::new();
+    let mut chat_author_ids = Vec::new();
+    // message_ids var is used for writing message info to files
+    let mut message_ids: Vec<MsgId> = Vec::new();
+    let mut message_json: Vec<ChatItemJSON> = Vec::new();
+    let mut referenced_external_messages: Vec<ChatItemJSON> = Vec::new();
+
+    for item in get_chat_msgs(context, chat_id, DC_GCM_ADDDAYMARKER, None).await {
+        if let Some(json_item) = match item {
+            ChatItem::Message { msg_id } => match Message::load_from_db(context, msg_id).await {
+                Ok(message) => {
+                    let filename = message.get_filename();
+                    if let Some(file) = filename {
+                        // push referenced blobs (attachments)
+                        blobs.push(file);
+                    }
+                    message_ids.push(message.id);
+                    // populate contactid list
+                    chat_author_ids.push(message.from_id);
+
+                    if let Ok(Some(ex_msg)) = message.quoted_message(&context).await {
+                        if ex_msg.get_chat_id() != chat_id {
+                            // if external add it to the file
+                            referenced_external_messages.push(ChatItemJSON::from_message(&ex_msg, &context).await)
+                            // contacts don't need to be referenced, because these should only be private replies
+                        }
+                    }
+
+                    Some(ChatItemJSON::from_message(&message, &context).await)
+                }
+                Err(error_message) => Some(ChatItemJSON::MessageError {
+                    id: msg_id.to_u32(),
+                    error: error_message.to_string(),
+                }),
+            },
+            ChatItem::DayMarker { timestamp } => Some(ChatItemJSON::DayMarker { timestamp }),
+            ChatItem::Marker1 => None,
+        } {
+            message_json.push(json_item)
+        }
+    }
+
+    // deduplicate contact list and load the contacts
+    chat_author_ids.sort();
+    chat_author_ids.dedup();
+    // load information about the authors
+    let mut chat_authors: HashMap<u32, ContactJSON> = HashMap::new();
+    chat_authors.insert(
+        0,
+        ContactJSON {
+            name: "Err: Contact not found".to_owned(),
+            email: "error@localhost".to_owned(),
+            profile_img: None,
+            color: "grey".to_owned(),
+        },
+    );
+    for author_id in chat_author_ids {
+        let contact = Contact::get_by_id(context, author_id).await;
+        if let Ok(c) = contact {
+            let profile_img_path: String;
+            if let Some(path) = c.get_profile_image(context).await {
+                profile_img_path = path
+                    .file_name()
+                    .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                    .to_str()
+                    .unwrap()
+                    .to_owned();
+                // push referenced blobs (avatars)
+                blobs.push(profile_img_path.clone());
+            } else {
+                profile_img_path = "".to_owned();
+            }
+            chat_authors.insert(
+                author_id,
+                ContactJSON {
+                    name: c.get_display_name().to_owned(),
+                    email: c.get_addr().to_owned(),
+                    profile_img: match profile_img_path != "" {
+                        true => Some(profile_img_path),
+                        false => None,
+                    },
+                    color: format!("{:#}", c.get_color()), // TODO
+                },
+            );
+        }
+    }
+
+    // Load information about the chat
+    let chat: Chat = Chat::load_from_db(context, chat_id).await.unwrap();
+    let chat_avatar = match chat.get_profile_image(context).await {
+        Some(img) => {
+            let path = img
+                .file_name()
+                .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                .to_str()
+                .unwrap()
+                .to_owned();
+            blobs.push(path.clone());
+            Some(format!("blobs/{}", path))
+        }
+        None => None,
+    };
+
+    let chat_json = ChatJSON {
+        chat_json_version: 1,
+        export_timestamp: time(),
+        name: chat.get_name().to_owned(),
+        color: format!("{:#}", chat.get_color(&context).await),
+        profile_img: chat_avatar,
+        contacts: chat_authors,
+        referenced_external_messages,
+        messages: message_json,
+        locations: crate::location::get_range(&context, chat_id, 0, 0, crate::dc_tools::time())
+            .await,
+    };
+
+    blobs.sort();
+    blobs.dedup();
+    ExportChatResult {
+        chat_json: serde_json::to_string(&chat_json).unwrap(),
+        message_ids,
+        referenced_blobs: blobs,
+    }
+}

src/format_flowed.rs

@@ -146,6 +146,17 @@ mod tests {
 
         let text = "> Not a quote";
         assert_eq!(format_flowed(text), " > Not a quote");
+
+        // Test space stuffing of wrapped lines
+        let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    >                               \n\
+                    > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "\x20> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        clients.\r\n\
+                        \x20>\r\n\
+                        \x20> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
     }
 
     #[test]

src/html.rs

@@ -426,7 +426,7 @@ test some special html-characters as < > and & but also " and &#x
     #[async_std::test]
     async fn test_get_html_empty() {
         let t = TestContext::new().await;
-        let msg_id = MsgId::new_unset();
+        let msg_id = MsgId::new(100);
         assert!(msg_id.get_html(&t).await.unwrap().is_none())
     }
 

src/imap.rs

@@ -521,29 +521,21 @@ impl Imap {
         // Write collected UIDs to SQLite database.
         context
             .sql
-            .transaction(|conn| {
-                Box::pin(async move {
-                    sqlx::query("UPDATE msgs SET server_uid=0 WHERE server_folder=?")
-                        .bind(&folder)
-                        .execute(&mut *conn)
-                        .await?;
-
-                    for (uid, rfc724_mid) in &msg_ids {
-                        // This may detect previously undetected moved
-                        // messages, so we update server_folder too.
-                        sqlx::query(
-                            "UPDATE msgs \
+            .transaction(move |transaction| {
+                transaction.execute(
+                    "UPDATE msgs SET server_uid=0 WHERE server_folder=?",
+                    params![folder],
+                )?;
+                for (uid, rfc724_mid) in &msg_ids {
+                    // This may detect previously undetected moved
+                    // messages, so we update server_folder too.
+                    transaction.execute(
+                        "UPDATE msgs \
                              SET server_folder=?,server_uid=? WHERE rfc724_mid=?",
-                        )
-                        .bind(&folder)
-                        .bind(uid)
-                        .bind(rfc724_mid)
-                        .execute(&mut *conn)
-                        .await?;
-                    }
-
-                    Ok(())
-                })
+                        params![folder, uid, rfc724_mid],
+                    )?;
+                }
+                Ok(())
             })
             .await?;
         Ok(())
@@ -718,7 +710,7 @@ impl Imap {
         }
 
         let (largest_uid_processed, error_cnt) = self
-            .fetch_many_msgs(context, &folder, uids, fetch_existing_msgs)
+            .fetch_many_msgs(context, folder, uids, fetch_existing_msgs)
             .await;
         read_errors += error_cnt;
 
@@ -866,10 +858,10 @@ impl Imap {
     /// Fetches a list of messages by server UID.
     ///
     /// Returns the last uid fetch successfully and an error count.
-    async fn fetch_many_msgs<S: AsRef<str>>(
+    async fn fetch_many_msgs(
         &mut self,
         context: &Context,
-        folder: S,
+        folder: &str,
         server_uids: Vec<u32>,
         fetching_existing_messages: bool,
     ) -> (Option<u32>, usize) {
@@ -907,14 +899,14 @@ impl Imap {
                         context,
                         "Error on fetching messages #{} from folder \"{}\"; error={}.",
                         &set,
-                        folder.as_ref(),
+                        folder,
                         err
                     );
                     return (None, server_uids.len());
                 }
             };
 
-            let folder = folder.as_ref().to_string();
+            let folder = folder.to_string();
 
             while let Some(Ok(msg)) = msgs.next().await {
                 let server_uid = msg.uid.unwrap_or_default();
@@ -933,7 +925,11 @@ impl Imap {
 
                 let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
                 if is_deleted || msg.body().is_none() {
-                    // No need to process these.
+                    info!(
+                        context,
+                        "Not processing deleted or empty msg {}", server_uid
+                    );
+                    last_uid = Some(server_uid);
                     continue;
                 }
 
@@ -1137,7 +1133,7 @@ impl Imap {
                 return Some(ImapActionResult::RetryLater);
             }
         }
-        match self.select_folder(context, Some(&folder)).await {
+        match self.select_folder(context, Some(folder)).await {
             Ok(_) => None,
             Err(select_folder::Error::ConnectionLost) => {
                 warn!(context, "Lost imap connection");
@@ -1732,15 +1728,9 @@ pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32)
     context
         .sql
         .execute(
-            sqlx::query(
-                "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?)
+            "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?)
                 ON CONFLICT(folder) DO UPDATE SET uid_next=? WHERE folder=?;",
-            )
-            .bind(folder)
-            .bind(0i32)
-            .bind(uid_next as i64)
-            .bind(uid_next as i64)
-            .bind(folder),
+            paramsv![folder, 0u32, uid_next, uid_next, folder],
         )
         .await?;
     Ok(())
@@ -1754,7 +1744,10 @@ pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32)
 async fn get_uid_next(context: &Context, folder: &str) -> Result<u32> {
     Ok(context
         .sql
-        .query_get_value(sqlx::query("SELECT uid_next FROM imap_sync WHERE folder=?;").bind(folder))
+        .query_get_value(
+            "SELECT uid_next FROM imap_sync WHERE folder=?;",
+            paramsv![folder],
+        )
         .await?
         .unwrap_or(0))
 }
@@ -1767,15 +1760,9 @@ pub(crate) async fn set_uidvalidity(
     context
         .sql
         .execute(
-            sqlx::query(
-                "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?)
+            "INSERT INTO imap_sync (folder, uidvalidity, uid_next) VALUES (?,?,?)
                 ON CONFLICT(folder) DO UPDATE SET uidvalidity=? WHERE folder=?;",
-            )
-            .bind(folder)
-            .bind(uidvalidity as i32)
-            .bind(0i32)
-            .bind(uidvalidity as i32)
-            .bind(folder),
+            paramsv![folder, uidvalidity, 0u32, uidvalidity, folder],
         )
         .await?;
     Ok(())
@@ -1785,7 +1772,8 @@ async fn get_uidvalidity(context: &Context, folder: &str) -> Result<u32> {
     Ok(context
         .sql
         .query_get_value(
-            sqlx::query("SELECT uidvalidity FROM imap_sync WHERE folder=?;").bind(folder),
+            "SELECT uidvalidity FROM imap_sync WHERE folder=?;",
+            paramsv![folder],
         )
         .await?
         .unwrap_or(0))

src/imap/client.rs

@@ -32,10 +32,10 @@ impl DerefMut for Client {
 }
 
 impl Client {
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+    pub async fn login(
         self,
-        username: U,
-        password: P,
+        username: &str,
+        password: &str,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session = inner
@@ -53,10 +53,10 @@ impl Client {
         Ok(Session { inner: session })
     }
 
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+    pub async fn authenticate(
         self,
-        auth_type: S,
-        authenticator: A,
+        auth_type: &str,
+        authenticator: impl async_imap::Authenticator,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session =
@@ -75,15 +75,14 @@ impl Client {
         Ok(Session { inner: session })
     }
 
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
+    pub async fn connect_secure(
+        addr: impl net::ToSocketAddrs,
+        domain: &str,
         strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
         let tls = dc_build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(domain.as_ref(), stream).await?);
+        let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
         let mut client = ImapClient::new(tls_stream);
 
         let _greeting = client
@@ -97,7 +96,7 @@ impl Client {
         })
     }
 
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> ImapResult<Self> {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
@@ -112,7 +111,7 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
+    pub async fn secure(self, domain: &str, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
@@ -121,7 +120,7 @@ impl Client {
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();
-            let ssl_stream = tls.connect(domain.as_ref(), stream).await?;
+            let ssl_stream = tls.connect(domain, stream).await?;
             let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
 
             Ok(Client {

src/imap/idle.rs

@@ -27,7 +27,7 @@ impl Imap {
         }
         self.setup_handle(context).await?;
 
-        self.select_folder(context, watch_folder.clone()).await?;
+        self.select_folder(context, watch_folder.as_deref()).await?;
 
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();

src/imap/scan_folders.rs

@@ -25,7 +25,7 @@ impl Imap {
         }
         info!(context, "Starting full folder scan");
 
-        self.setup_handle(context).await?;
+        self.connect_configured(context).await?;
         let session = self.session.as_mut();
         let session = session.context("scan_folders(): IMAP No Connection established")?;
         let folders: Vec<_> = session.list(Some(""), Some("*")).await?.collect().await;
@@ -61,14 +61,8 @@ impl Imap {
                 spam_folder = Some(folder.name().to_string());
             }
 
-            if watched_folders.contains(&foldername.to_string()) {
-                info!(
-                    context,
-                    "Not scanning folder {} as it is watched anyway", foldername
-                );
-            } else {
-                info!(context, "Scanning folder: {}", foldername);
-
+            // Don't scan folders that are watched anyway
+            if !watched_folders.contains(&foldername.to_string()) {
                 if let Err(e) = self.fetch_new_messages(context, foldername, false).await {
                     warn!(context, "Can't fetch new msgs in scanned folder: {:#}", e);
                 }

src/imap/select_folder.rs

@@ -62,10 +62,10 @@ impl Imap {
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     /// Returns whether a new folder was selected.
-    pub(super) async fn select_folder<S: AsRef<str>>(
+    pub(super) async fn select_folder(
         &mut self,
         context: &Context,
-        folder: Option<S>,
+        folder: Option<&str>,
     ) -> Result<NewlySelected> {
         if self.session.is_none() {
             self.config.selected_folder = None;
@@ -76,9 +76,9 @@ impl Imap {
 
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
-        if let Some(ref folder) = folder {
+        if let Some(folder) = folder {
             if let Some(ref selected_folder) = self.config.selected_folder {
-                if folder.as_ref() == selected_folder {
+                if folder == selected_folder {
                     return Ok(NewlySelected::No);
                 }
             }
@@ -88,7 +88,7 @@ impl Imap {
         self.maybe_close_folder(context).await?;
 
         // select new folder
-        if let Some(ref folder) = folder {
+        if let Some(folder) = folder {
             if let Some(ref mut session) = &mut self.session {
                 let res = session.select(folder).await;
 
@@ -98,7 +98,7 @@ impl Imap {
 
                 match res {
                     Ok(mailbox) => {
-                        self.config.selected_folder = Some(folder.as_ref().to_string());
+                        self.config.selected_folder = Some(folder.to_string());
                         self.config.selected_mailbox = Some(mailbox);
                         Ok(NewlySelected::Yes)
                     }
@@ -108,7 +108,7 @@ impl Imap {
                         Err(Error::ConnectionLost)
                     }
                     Err(async_imap::error::Error::Validate(_)) => {
-                        Err(Error::BadFolderName(folder.as_ref().to_string()))
+                        Err(Error::BadFolderName(folder.to_string()))
                     }
                     Err(err) => {
                         self.config.selected_folder = None;

src/imex.rs

@@ -3,17 +3,18 @@
 use std::any::Any;
 use std::ffi::OsStr;
 
+use ::pgp::types::KeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
-use async_std::path::{Path, PathBuf};
 use async_std::{
     fs::{self, File},
+    path::{Path, PathBuf},
     prelude::*,
 };
+use async_tar::Archive;
 use rand::{thread_rng, Rng};
-use sqlx::Row;
 
-use crate::chat;
-use crate::chat::delete_and_reset_all_device_msgs;
+use crate::blob::BlobObject;
+use crate::chat::{self, delete_and_reset_all_device_msgs, ChatId};
 use crate::config::Config;
 use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
 use crate::context::Context;
@@ -24,15 +25,13 @@ use crate::dc_tools::{
 use crate::e2ee;
 use crate::events::EventType;
 use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
+use crate::log::LogExt;
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Param;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock_str;
-use crate::{blob::BlobObject, log::LogExt};
-use ::pgp::types::KeyTrait;
-use async_tar::Archive;
 
 // Name of the database file in the backup.
 const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
@@ -79,7 +78,7 @@ pub enum ImexMode {
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
-pub async fn imex(context: &Context, what: ImexMode, param1: impl AsRef<Path>) -> Result<()> {
+pub async fn imex(context: &Context, what: ImexMode, param1: &Path) -> Result<()> {
     let cancel = context.alloc_ongoing().await?;
 
     let res = async {
@@ -124,8 +123,7 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
-    let dir_name = dir_name.as_ref();
+pub async fn has_backup(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_name = "".to_string();
     let mut newest_backup_path: Option<PathBuf> = None;
@@ -155,8 +153,7 @@ pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result
 }
 
 /// Returns the filename of the backup found (otherwise an error)
-pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
-    let dir_name = dir_name.as_ref();
+pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String> {
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
     let mut newest_backup_name = "".to_string();
@@ -232,7 +229,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     )
     .await?;
 
-    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF).await?;
+    let chat_id = ChatId::create_for_contact(context, DC_CONTACT_ID_SELF).await?;
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
@@ -451,8 +448,8 @@ pub fn normalize_setup_code(s: &str) -> String {
     out
 }
 
-async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -> Result<()> {
-    info!(context, "Import/export dir: {}", path.as_ref().display());
+async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<()> {
+    info!(context, "Import/export dir: {}", path.display());
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
@@ -476,12 +473,8 @@ async fn imex_inner(context: &Context, what: ImexMode, path: impl AsRef<Path>) -
 }
 
 /// Import Backup
-async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
-    if backup_to_import
-        .as_ref()
-        .to_string_lossy()
-        .ends_with(".bak")
-    {
+async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> {
+    if backup_to_import.to_string_lossy().ends_with(".bak") {
         // Backwards compability
         return import_backup_old(context, backup_to_import).await;
     }
@@ -489,7 +482,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     info!(
         context,
         "Import \"{}\" to \"{}\".",
-        backup_to_import.as_ref().display(),
+        backup_to_import.display(),
         context.get_dbfile().display()
     );
 
@@ -547,7 +540,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 
     context
         .sql
-        .open(context, &context.get_dbfile(), false)
+        .open(context, context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
@@ -556,11 +549,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     Ok(())
 }
 
-async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
-        backup_to_import.as_ref().display(),
+        backup_to_import.display(),
         context.get_dbfile().display()
     );
 
@@ -580,14 +573,14 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     );
 
     ensure!(
-        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()).await,
+        dc_copy_file(context, backup_to_import, context.get_dbfile()).await,
         "could not copy file"
     );
     /* error already logged */
     /* re-open copied database file */
     context
         .sql
-        .open(context, &context.get_dbfile(), false)
+        .open(context, context.get_dbfile(), false)
         .await
         .context("Could not re-open db")?;
 
@@ -595,9 +588,8 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
 
     let total_files_cnt = context
         .sql
-        .count(sqlx::query("SELECT COUNT(*) FROM backup_blobs;"))
+        .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![])
         .await?;
-
     info!(
         context,
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
@@ -607,25 +599,33 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
     // consuming too much memory.
     let file_ids = context
         .sql
-        .fetch(sqlx::query("SELECT id FROM backup_blobs ORDER BY id"))
-        .await?
-        .map(|row| row?.try_get(0))
-        .collect::<sqlx::Result<Vec<i64>>>()
+        .query_map(
+            "SELECT id FROM backup_blobs ORDER BY id",
+            paramsv![],
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
+                    .map_err(Into::into)
+            },
+        )
         .await?;
 
     let mut all_files_extracted = true;
     for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
         // Load a single blob into memory
-        let row = context
+        let (file_name, file_blob) = context
             .sql
-            .fetch_one(
-                sqlx::query("SELECT file_name, file_content FROM backup_blobs WHERE id = ?")
-                    .bind(file_id),
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| {
+                    let file_name: String = row.get(0)?;
+                    let file_blob: Vec<u8> = row.get(1)?;
+                    Ok((file_name, file_blob))
+                },
             )
             .await?;
 
-        let file_name: String = row.try_get(0)?;
-        let file_blob: &[u8] = row.try_get(1)?;
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -643,16 +643,16 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
         }
 
         let path_filename = context.get_blobdir().join(file_name);
-        dc_write_file(context, &path_filename, file_blob).await?;
+        dc_write_file(context, &path_filename, &file_blob).await?;
     }
 
     if all_files_extracted {
         // only delete backup_blobs if all files were successfully extracted
         context
             .sql
-            .execute(sqlx::query("DROP TABLE backup_blobs;"))
+            .execute("DROP TABLE backup_blobs;", paramsv![])
             .await?;
-        context.sql.execute(sqlx::query("VACUUM;")).await.ok();
+        context.sql.execute("VACUUM;", paramsv![]).await.ok();
         Ok(())
     } else {
         bail!("received stop signal");
@@ -663,7 +663,7 @@ async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>
  * Export backup
  ******************************************************************************/
 #[allow(unused)]
-async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_backup(context: &Context, dir: &Path) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     let now = time();
     let (temp_path, dest_path) = get_next_backup_path(dir, now).await?;
@@ -677,7 +677,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
 
     context
         .sql
-        .execute(sqlx::query("VACUUM;"))
+        .execute("VACUUM;", paramsv![])
         .await
         .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
 
@@ -699,10 +699,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     let res = export_backup_inner(context, &temp_path).await;
 
     // we re-open the database after export is finished
-    context
-        .sql
-        .open(context, &context.get_dbfile(), false)
-        .await;
+    context.sql.open(context, context.get_dbfile(), false).await;
 
     match &res {
         Ok(_) => {
@@ -769,7 +766,7 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 /*******************************************************************************
  * Classic key import
  ******************************************************************************/
-async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
     /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
     plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
     Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
@@ -779,12 +776,12 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
     let mut set_default: bool;
     let mut imported_cnt = 0;
 
-    let dir_name = dir.as_ref().to_string_lossy();
+    let dir_name = dir.to_string_lossy();
     let mut dir_handle = async_std::fs::read_dir(&dir).await?;
     while let Some(entry) = dir_handle.next().await {
         let entry_fn = entry?.file_name();
         let name_f = entry_fn.to_string_lossy();
-        let path_plus_name = dir.as_ref().join(&entry_fn);
+        let path_plus_name = dir.join(&entry_fn);
         match dc_get_filesuffix_lc(&name_f) {
             Some(suffix) => {
                 if suffix != "asc" {
@@ -827,32 +824,35 @@ async fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
     Ok(())
 }
 
-async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let mut export_errors = 0;
 
-    let mut keys = context
+    let keys = context
         .sql
-        .fetch(sqlx::query(
+        .query_map(
             "SELECT id, public_key, private_key, is_default FROM keypairs;",
-        ))
-        .await?
-        .map(|row| -> sqlx::Result<_> {
-            let row = row?;
-            let id = row.try_get(0)?;
-            let public_key_blob: &[u8] = row.try_get(1)?;
-            let public_key = SignedPublicKey::from_slice(public_key_blob);
-            let private_key_blob: &[u8] = row.try_get(2)?;
-            let private_key = SignedSecretKey::from_slice(private_key_blob);
-            let is_default: i32 = row.try_get(3)?;
+            paramsv![],
+            |row| {
+                let id = row.get(0)?;
+                let public_key_blob: Vec<u8> = row.get(1)?;
+                let public_key = SignedPublicKey::from_slice(&public_key_blob);
+                let private_key_blob: Vec<u8> = row.get(2)?;
+                let private_key = SignedSecretKey::from_slice(&private_key_blob);
+                let is_default: i32 = row.get(3)?;
 
-            Ok((id, public_key, private_key, is_default))
-        });
+                Ok((id, public_key, private_key, is_default))
+            },
+            |keys| {
+                keys.collect::<std::result::Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await?;
 
-    while let Some(parts) = keys.next().await {
-        let (id, public_key, private_key, is_default) = parts?;
+    for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default != 0);
         if let Ok(key) = public_key {
-            if export_key_to_asc_file(context, &dir, id, &key)
+            if export_key_to_asc_file(context, dir, id, &key)
                 .await
                 .is_err()
             {
@@ -862,7 +862,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
             export_errors += 1;
         }
         if let Ok(key) = private_key {
-            if export_key_to_asc_file(context, &dir, id, &key)
+            if export_key_to_asc_file(context, dir, id, &key)
                 .await
                 .is_err()
             {
@@ -882,7 +882,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
  ******************************************************************************/
 async fn export_key_to_asc_file<T>(
     context: &Context,
-    dir: impl AsRef<Path>,
+    dir: &Path,
     id: Option<i64>,
     key: &T,
 ) -> std::io::Result<()>
@@ -899,7 +899,7 @@ where
             "unknown"
         };
         let id = id.map_or("default".into(), |i| i.to_string());
-        dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
+        dir.join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(
         context,
@@ -979,7 +979,7 @@ mod tests {
     async fn test_export_public_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().public;
-        let blobdir = "$BLOBDIR";
+        let blobdir = Path::new("$BLOBDIR");
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
             .is_ok());
@@ -994,7 +994,7 @@ mod tests {
     async fn test_export_private_key_to_asc_file() {
         let context = TestContext::new().await;
         let key = alice_keypair().secret;
-        let blobdir = "$BLOBDIR";
+        let blobdir = Path::new("$BLOBDIR");
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
             .is_ok());
@@ -1009,7 +1009,7 @@ mod tests {
     async fn test_export_and_import_key() {
         let context = TestContext::new().await;
         context.configure_alice().await;
-        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let blobdir = context.ctx.get_blobdir();
         if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir).await {
             panic!("got error on export: {:?}", err);
         }

src/job.rs

@@ -7,37 +7,37 @@ use std::{fmt, time::Duration};
 
 use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
 use async_smtp::smtp::response::{Category, Code, Detail};
-use async_std::prelude::*;
 use async_std::task::sleep;
+use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
-use sqlx::Row;
 
+use crate::blob::BlobObject;
+use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ChatItem};
+use crate::config::Config;
+use crate::constants::{Blocked, Chattype, DC_CHAT_ID_DEADDROP};
+use crate::contact::{normalize_name, Contact, Modifier, Origin};
+use crate::context::Context;
 use crate::dc_tools::{dc_delete_file, dc_read_file, time};
 use crate::ephemeral::load_imap_deletion_msgid;
 use crate::events::EventType;
 use crate::imap::{Imap, ImapActionResult};
 use crate::location;
-use crate::message::MsgId;
-use crate::message::{self, Message, MessageState};
+use crate::log::LogExt;
+use crate::message::{self, Message, MessageState, MsgId};
 use crate::mimefactory::MimeFactory;
 use crate::param::{Param, Params};
+use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
-use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
-use crate::{
-    chat::{self, Chat, ChatId, ChatItem},
-    constants::DC_CHAT_ID_DEADDROP,
-};
-use crate::{config::Config, constants::Blocked};
-use crate::{constants::Chattype, contact::Contact};
-use crate::{context::Context, log::LogExt};
-use crate::{scheduler::InterruptInfo, sql};
+use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
 
 /// Thread IDs
-#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
+#[derive(
+    Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u32)]
 pub(crate) enum Thread {
     Unknown = 0,
@@ -75,7 +75,17 @@ impl Default for Thread {
 }
 
 #[derive(
-    Debug, Display, Copy, Clone, PartialEq, Eq, PartialOrd, FromPrimitive, ToPrimitive, sqlx::Type,
+    Debug,
+    Display,
+    Copy,
+    Clone,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    FromPrimitive,
+    ToPrimitive,
+    FromSql,
+    ToSql,
 )]
 #[repr(u32)]
 pub enum Action {
@@ -130,7 +140,7 @@ impl From<Action> for Thread {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct Job {
     pub job_id: u32,
     pub action: Action,
@@ -173,7 +183,7 @@ impl Job {
         if self.job_id != 0 {
             context
                 .sql
-                .execute(sqlx::query("DELETE FROM jobs WHERE id=?;").bind(self.job_id as i32))
+                .execute("DELETE FROM jobs WHERE id=?;", paramsv![self.job_id as i32])
                 .await?;
         }
 
@@ -192,24 +202,26 @@ impl Job {
             context
                 .sql
                 .execute(
-                    sqlx::query(
-                        "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
-                    )
-                    .bind(self.desired_timestamp)
-                    .bind(self.tries as i64)
-                    .bind(self.param.to_string())
-                    .bind(self.job_id as i32),
+                    "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+                    paramsv![
+                        self.desired_timestamp,
+                        self.tries as i64,
+                        self.param.to_string(),
+                        self.job_id as i32,
+                    ],
                 )
                 .await?;
         } else {
             context.sql.execute(
-                sqlx::query("INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);")
-                    .bind(self.added_timestamp)
-                    .bind(thread)
-                    .bind(self.action)
-                    .bind(self.foreign_id)
-                    .bind(self.param.to_string())
-                    .bind(self.desired_timestamp)
+                "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
+                paramsv![
+                    self.added_timestamp,
+                    thread,
+                    self.action,
+                    self.foreign_id,
+                    self.param.to_string(),
+                    self.desired_timestamp
+                ]
             ).await?;
         }
 
@@ -417,32 +429,39 @@ impl Job {
         &self,
         context: &Context,
         contact_id: u32,
-    ) -> sql::Result<(Vec<u32>, Vec<String>)> {
+    ) -> Result<(Vec<u32>, Vec<String>)> {
         // Extract message IDs from job parameters
-        let mut rows = context
+        let res: Vec<(u32, MsgId)> = context
             .sql
-            .fetch(
-                sqlx::query("SELECT id, param FROM jobs WHERE foreign_id=? AND id!=?")
-                    .bind(contact_id)
-                    .bind(self.job_id),
+            .query_map(
+                "SELECT id, param FROM jobs WHERE foreign_id=? AND id!=?",
+                paramsv![contact_id, self.job_id],
+                |row| {
+                    let job_id: u32 = row.get(0)?;
+                    let params_str: String = row.get(1)?;
+                    let params: Params = params_str.parse().unwrap_or_default();
+                    Ok((job_id, params))
+                },
+                |jobs| {
+                    let res = jobs
+                        .filter_map(|row| {
+                            let (job_id, params) = row.ok()?;
+                            let msg_id = params.get_msg_id()?;
+                            Some((job_id, msg_id))
+                        })
+                        .collect();
+                    Ok(res)
+                },
             )
             .await?;
 
         // Load corresponding RFC724 message IDs
         let mut job_ids = Vec::new();
         let mut rfc724_mids = Vec::new();
-
-        while let Some(row) = rows.next().await {
-            let row = row?;
-            let job_id: u32 = row.try_get(0)?;
-            let params_str: String = row.try_get(1)?;
-            let params: Params = params_str.parse().unwrap_or_default();
-            if let Some(msg_id) = params.get_msg_id() {
-                if let Ok(Message { rfc724_mid, .. }) = Message::load_from_db(context, msg_id).await
-                {
-                    job_ids.push(job_id);
-                    rfc724_mids.push(rfc724_mid);
-                }
+        for (job_id, msg_id) in res {
+            if let Ok(Message { rfc724_mid, .. }) = Message::load_from_db(context, msg_id).await {
+                job_ids.push(job_id);
+                rfc724_mids.push(rfc724_mid);
             }
         }
         Ok((job_ids, rfc724_mids))
@@ -635,7 +654,6 @@ impl Job {
                 // Hidden messages are similar to trashed, but are
                 // related to some chat. We also delete their
                 // database records.
-                info!(context, "verbose (issue 2335): will delete from db");
                 job_try!(msg.id.delete_from_db(context).await)
             } else {
                 // Remove server UID from the database record.
@@ -646,7 +664,6 @@ impl Job {
                 // we remove UID to reduce the number of messages
                 // pointing to the corresponding UID. Once the counter
                 // reaches zero, we will remove the message.
-                info!(context, "verbose (issue 2335): will unlink");
                 job_try!(msg.id.unlink(context).await);
             }
             Status::Finished(Ok(()))
@@ -711,10 +728,12 @@ impl Job {
                 };
                 match chat.typ {
                     Chattype::Group | Chattype::Mailinglist => {
-                        if let Ok((_1to1_chat, Blocked::Not)) =
-                            chat::lookup_by_contact_id(context, msg.from_id).await
+                        if let Ok(Some(one_to_one_chat)) =
+                            ChatIdBlocked::lookup_by_contact(context, msg.from_id).await
                         {
-                            chat.id.unblock(context).await;
+                            if one_to_one_chat.blocked == Blocked::Not {
+                                chat.id.unblock(context).await;
+                            }
                         }
                     }
                     Chattype::Single | Chattype::Undefined => {}
@@ -822,29 +841,31 @@ impl Job {
 pub async fn kill_action(context: &Context, action: Action) -> bool {
     context
         .sql
-        .execute(sqlx::query("DELETE FROM jobs WHERE action=?;").bind(action))
+        .execute("DELETE FROM jobs WHERE action=?;", paramsv![action])
         .await
         .is_ok()
 }
 
 /// Remove jobs with specified IDs.
-async fn kill_ids(context: &Context, job_ids: &[u32]) -> sql::Result<()> {
+async fn kill_ids(context: &Context, job_ids: &[u32]) -> Result<()> {
     let q = format!(
         "DELETE FROM jobs WHERE id IN({})",
         job_ids.iter().map(|_| "?").join(",")
     );
-    let mut query = sqlx::query(&q);
-    for id in job_ids {
-        query = query.bind(*id);
-    }
-    context.sql.execute(query).await?;
+    context
+        .sql
+        .execute(q, rusqlite::params_from_iter(job_ids))
+        .await?;
     Ok(())
 }
 
 pub async fn action_exists(context: &Context, action: Action) -> bool {
     context
         .sql
-        .exists(sqlx::query("SELECT COUNT(*) FROM jobs WHERE action=?;").bind(action))
+        .exists(
+            "SELECT COUNT(*) FROM jobs WHERE action=?;",
+            paramsv![action],
+        )
         .await
         .unwrap_or_default()
 }
@@ -853,7 +874,7 @@ async fn set_delivered(context: &Context, msg_id: MsgId) -> Result<()> {
     message::update_msg_state(context, msg_id, MessageState::OutDelivered).await;
     let chat_id: ChatId = context
         .sql
-        .query_get_value(sqlx::query("SELECT chat_id FROM msgs WHERE id=?").bind(msg_id))
+        .query_get_value("SELECT chat_id FROM msgs WHERE id=?", paramsv![msg_id])
         .await?
         .unwrap_or_default();
     context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
@@ -883,8 +904,8 @@ async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, fold
 
                 match Contact::add_or_lookup(
                     context,
-                    display_name_normalized,
-                    contact.addr,
+                    &display_name_normalized,
+                    &contact.addr,
                     Origin::OutgoingTo,
                 )
                 .await
@@ -1030,9 +1051,8 @@ pub(crate) enum Connection<'a> {
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
+pub(crate) async fn load_imap_deletion_job(context: &Context) -> Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
-        info!(context, "verbose (issue 2335): loading imap deletion job");
         Some(Job::new(
             Action::DeleteMsgOnImap,
             msg_id.to_u32(),
@@ -1118,7 +1138,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_
             if let Err(err) = res {
                 warn!(
                     context,
-                    "{} removes job {} as it failed with error {:?}", &connection, job, err
+                    "{} removes job {} as it failed with error {:#}", &connection, job, err
                 );
             } else {
                 info!(
@@ -1142,7 +1162,7 @@ async fn perform_job_action(
 ) -> Status {
     info!(
         context,
-        "{} begin immediate try {} of job {:?} - verbose (issue 2335)", &connection, tries, job
+        "{} begin immediate try {} of job {}", &connection, tries, job
     );
 
     let try_res = match job.action {
@@ -1285,77 +1305,65 @@ pub(crate) async fn load_next(
         sleep(Duration::from_millis(500)).await;
     }
 
+    let query;
+    let params;
     let t = time();
+    let m;
     let thread_i = thread as i64;
 
-    let get_query = || {
-        if let Some(msg_id) = info.msg_id {
-            sqlx::query(
-                r#"
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND foreign_id=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-            .bind(msg_id)
-        } else if !info.probe_network {
-            // processing for first-try and after backoff-timeouts:
-            // process jobs in the order they were added.
-            sqlx::query(
-                r#"
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
+        // processing for first-try and after backoff-timeouts:
+        // process jobs in the order they were added.
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-            .bind(t)
-        } else {
-            // processing after call to dc_maybe_network():
-            // process _all_ pending jobs that failed before
-            // in the order of their backoff-times.
-            sqlx::query(
-                r#"
+"#;
+        params = paramsv![thread_i, t];
+    } else {
+        // processing after call to dc_maybe_network():
+        // process _all_ pending jobs that failed before
+        // in the order of their backoff-times.
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#,
-            )
-            .bind(thread_i)
-        }
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
         let job_res = context
             .sql
-            .fetch_optional(get_query())
-            .await
-            .and_then(|row| {
-                if let Some(row) = row {
-                    Ok(Some(Job {
-                        job_id: row.try_get("id")?,
-                        action: row.try_get("action")?,
-                        foreign_id: row.try_get("foreign_id")?,
-                        desired_timestamp: row.try_get("desired_timestamp")?,
-                        added_timestamp: row.try_get("added_timestamp")?,
-                        tries: row.try_get::<i64, _>("tries")? as u32,
-                        param: row
-                            .try_get::<String, _>("param")?
-                            .parse()
-                            .unwrap_or_default(),
-                        pending_error: None,
-                    }))
-                } else {
-                    Ok(None)
-                }
-            });
+            .query_row_optional(query, params.clone(), |row| {
+                let job = Job {
+                    job_id: row.get("id")?,
+                    action: row.get("action")?,
+                    foreign_id: row.get("foreign_id")?,
+                    desired_timestamp: row.get("desired_timestamp")?,
+                    added_timestamp: row.get("added_timestamp")?,
+                    tries: row.get("tries")?,
+                    param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
+                    pending_error: None,
+                };
+
+                Ok(job)
+            })
+            .await;
 
         match job_res {
             Ok(job) => break job,
@@ -1366,14 +1374,13 @@ LIMIT 1;
                 // TODO: improve by only doing a single query
                 match context
                     .sql
-                    .fetch_one(get_query())
+                    .query_row(query, params.clone(), |row| row.get::<_, i32>(0))
                     .await
-                    .and_then(|row| row.try_get::<i32, _>(0).map_err(Into::into))
                 {
                     Ok(id) => {
                         if let Err(err) = context
                             .sql
-                            .execute(sqlx::query("DELETE FROM jobs WHERE id=?;").bind(id))
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
                             .await
                         {
                             warn!(context, "failed to delete job {}: {:?}", id, err);
@@ -1401,14 +1408,9 @@ LIMIT 1;
                         .unwrap_or_default()
                         .or(Some(job))
                 } else {
-                    info!(context, "verbose (issue 2335): executing job normally");
                     Some(job)
                 }
             } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
-                info!(
-                    context,
-                    "verbose (issue 2335): loaded imap deletion job (no others queued)"
-                );
                 Some(job)
             } else {
                 load_housekeeping_job(context).await
@@ -1429,17 +1431,17 @@ mod tests {
         context
             .sql
             .execute(
-                sqlx::query(
-                    "INSERT INTO jobs
+                "INSERT INTO jobs
                    (added_timestamp, thread, action, foreign_id, param, desired_timestamp)
                  VALUES (?, ?, ?, ?, ?, ?);",
-                )
-                .bind(now)
-                .bind(Thread::from(Action::MoveMsg))
-                .bind(if valid { Action::MoveMsg as i32 } else { -1 })
-                .bind(foreign_id)
-                .bind(Params::new().to_string())
-                .bind(now),
+                paramsv![
+                    now,
+                    Thread::from(Action::MoveMsg),
+                    if valid { Action::MoveMsg as i32 } else { -1 },
+                    foreign_id,
+                    Params::new().to_string(),
+                    now
+                ],
             )
             .await
             .unwrap();
@@ -1459,7 +1461,7 @@ mod tests {
         )
         .await;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
-        assert!(jobs.unwrap().action == Action::Housekeeping);
+        assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
         insert_job(&t, 1, true).await;
         let jobs = load_next(

src/key.rs

@@ -9,14 +9,12 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
-use sqlx::Row;
 use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
-use crate::sql;
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
@@ -32,8 +30,6 @@ pub enum Error {
     Pgp(#[from] pgp::errors::Error),
     #[error("Failed to generate PGP key: {}", _0)]
     Keygen(#[from] crate::pgp::PgpKeygenError),
-    #[error("Failed to load key: {}", _0)]
-    LoadKey(#[from] sql::Error),
     #[error("Failed to save generated key: {}", _0)]
     StoreKey(#[from] SaveKeyError),
     #[error("No address configured")]
@@ -42,8 +38,6 @@ pub enum Error {
     InvalidConfiguredAddr(#[from] InvalidEmailError),
     #[error("no data provided")]
     Empty,
-    #[error("db: {}", _0)]
-    Sql(#[from] sqlx::Error),
     #[error("{0}")]
     Other(#[from] anyhow::Error),
 }
@@ -123,17 +117,22 @@ impl DcKey for SignedPublicKey {
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
         match context
             .sql
-            .fetch_optional(sqlx::query(
+            .query_row_optional(
                 r#"
             SELECT public_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            ))
+                paramsv![],
+                |row| {
+                    let bytes: Vec<u8> = row.get(0)?;
+                    Ok(bytes)
+                },
+            )
             .await?
         {
-            Some(row) => Self::from_slice(row.try_get(0)?),
+            Some(bytes) => Self::from_slice(&bytes),
             None => {
                 let keypair = generate_keypair(context).await?;
                 Ok(keypair.public)
@@ -165,17 +164,22 @@ impl DcKey for SignedSecretKey {
     async fn load_self(context: &Context) -> Result<Self::KeyType> {
         match context
             .sql
-            .fetch_optional(sqlx::query(
+            .query_row_optional(
                 r#"
             SELECT private_key
               FROM keypairs
              WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
                AND is_default=1;
             "#,
-            ))
+                paramsv![],
+                |row| {
+                    let bytes: Vec<u8> = row.get(0)?;
+                    Ok(bytes)
+                },
+            )
             .await?
         {
-            Some(row) => Self::from_slice(row.try_get(0)?),
+            Some(bytes) => Self::from_slice(&bytes),
             None => {
                 let keypair = generate_keypair(context).await?;
                 Ok(keypair.secret)
@@ -228,23 +232,26 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     // Check if the key appeared while we were waiting on the lock.
     match context
         .sql
-        .fetch_optional(
-            sqlx::query(
-                r#"
+        .query_row_optional(
+            r#"
         SELECT public_key, private_key
           FROM keypairs
          WHERE addr=?1
            AND is_default=1;
         "#,
-            )
-            .bind(addr.to_string()),
+            paramsv![addr],
+            |row| {
+                let pub_bytes: Vec<u8> = row.get(0)?;
+                let sec_bytes: Vec<u8> = row.get(1)?;
+                Ok((pub_bytes, sec_bytes))
+            },
         )
         .await?
     {
-        Some(row) => Ok(KeyPair {
+        Some((pub_bytes, sec_bytes)) => Ok(KeyPair {
             addr,
-            public: SignedPublicKey::from_slice(row.try_get(0)?)?,
-            secret: SignedSecretKey::from_slice(row.try_get(1)?)?,
+            public: SignedPublicKey::from_slice(&pub_bytes)?,
+            secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         None => {
             let start = std::time::SystemTime::now();
@@ -319,16 +326,15 @@ pub async fn store_self_keypair(
     context
         .sql
         .execute(
-            sqlx::query("DELETE FROM keypairs WHERE public_key=? OR private_key=?;")
-                .bind(&public_key)
-                .bind(&secret_key),
+            "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
+            paramsv![public_key, secret_key],
         )
         .await
         .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
     if default == KeyPairUse::Default {
         context
             .sql
-            .execute(sqlx::query("UPDATE keypairs SET is_default=0;"))
+            .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
             .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
     }
@@ -343,15 +349,9 @@ pub async fn store_self_keypair(
     context
         .sql
         .execute(
-            sqlx::query(
-                "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
+            "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
                 VALUES (?,?,?,?,?);",
-            )
-            .bind(addr)
-            .bind(is_default)
-            .bind(&public_key)
-            .bind(&secret_key)
-            .bind(t),
+            paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
         .map_err(|err| SaveKeyError::new("failed to insert keypair", err))?;
@@ -625,7 +625,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 
         let nrows = || async {
             ctx.sql
-                .count(sqlx::query("SELECT COUNT(*) FROM keypairs;"))
+                .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
                 .await
                 .unwrap()
         };

src/lib.rs

@@ -1,11 +1,11 @@
+#![forbid(unsafe_code)]
 #![deny(
     clippy::correctness,
     missing_debug_implementations,
     clippy::all,
     clippy::indexing_slicing,
     clippy::wildcard_imports,
-    clippy::needless_borrow,
-    unsafe_code
+    clippy::needless_borrow
 )]
 #![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
@@ -13,10 +13,16 @@
 extern crate num_derive;
 #[macro_use]
 extern crate smallvec;
+#[macro_use]
+extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
 
+pub trait ToSql: rusqlite::ToSql + Send + Sync {}
+
+impl<T: rusqlite::ToSql + Send + Sync> ToSql for T {}
+
 #[macro_use]
 pub mod log;
 #[macro_use]
@@ -45,6 +51,7 @@ pub mod contact;
 pub mod context;
 mod e2ee;
 pub mod ephemeral;
+pub mod export_chat;
 mod imap;
 pub mod imex;
 mod scheduler;

src/location.rs

@@ -2,10 +2,8 @@
 use std::convert::TryFrom;
 
 use anyhow::{ensure, Error};
-use async_std::prelude::*;
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
-use sqlx::Row;
 
 use crate::chat::{self, ChatId};
 use crate::config::Config;
@@ -18,9 +16,8 @@ use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::Params;
 use crate::stock_str;
-
-/// Location record
-#[derive(Debug, Clone, Default)]
+use serde::Serialize;
+#[derive(Debug, Clone, Default, Serialize)]
 pub struct Location {
     pub location_id: u32,
     pub latitude: f64,
@@ -201,15 +198,15 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
         if context
             .sql
             .execute(
-                sqlx::query(
-                    "UPDATE chats    \
+                "UPDATE chats    \
              SET locations_send_begin=?,        \
              locations_send_until=?  \
              WHERE id=?",
-                )
-                .bind(if 0 != seconds { now } else { 0 })
-                .bind(if 0 != seconds { now + seconds } else { 0 })
-                .bind(chat_id),
+                paramsv![
+                    if 0 != seconds { now } else { 0 },
+                    if 0 != seconds { now + seconds } else { 0 },
+                    chat_id,
+                ],
             )
             .await
             .is_ok()
@@ -262,17 +259,16 @@ pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<Cha
         Some(chat_id) => context
             .sql
             .exists(
-                sqlx::query("SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;")
-                    .bind(chat_id)
-                    .bind(time()),
+                "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
             )
             .await
             .unwrap_or_default(),
         None => context
             .sql
             .exists(
-                sqlx::query("SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;")
-                    .bind(time()),
+                "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
             )
             .await
             .unwrap_or_default(),
@@ -285,29 +281,28 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
     }
     let mut continue_streaming = false;
 
-    if let Ok(mut chats) = context
+    if let Ok(chats) = context
         .sql
-        .fetch(sqlx::query("SELECT id FROM chats WHERE locations_send_until>?;").bind(time()))
+        .query_map(
+            "SELECT id FROM chats WHERE locations_send_until>?;",
+            paramsv![time()],
+            |row| row.get::<_, i32>(0),
+            |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
         .await
-        .map(|rows| rows.map(|row| row?.try_get::<i32, _>(0)))
     {
-        while let Some(chat_id) = chats.next().await {
-            let chat_id = match chat_id {
-                Ok(id) => id,
-                Err(_) => break,
-            };
+        for chat_id in chats {
             if let Err(err) = context.sql.execute(
-                sqlx::query(
                     "INSERT INTO locations  \
-                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);"
-                )
-                    .bind(latitude)
-                    .bind(longitude)
-                    .bind(accuracy)
-                    .bind(time())
-                    .bind(chat_id)
-                    .bind(DC_CONTACT_ID_SELF)
-
+                     (latitude, longitude, accuracy, timestamp, chat_id, from_id) VALUES (?,?,?,?,?,?);",
+                    paramsv![
+                        latitude,
+                        longitude,
+                        accuracy,
+                        time(),
+                        chat_id,
+                        DC_CONTACT_ID_SELF,
+                    ]
             ).await {
                 warn!(context, "failed to store location {:?}", err);
             } else {
@@ -342,50 +337,54 @@ pub async fn get_range(
         Some(contact_id) => (0, contact_id),
         None => (1, 0), // this contact_id is unused
     };
-
     let list = context
         .sql
-        .fetch(
-            sqlx::query(
-                "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
+        .query_map(
+            "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
              COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
              FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
              AND (? OR l.from_id=?) \
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
-            )
-            .bind(disable_chat_id)
-            .bind(chat_id)
-            .bind(disable_contact_id)
-            .bind(contact_id as i64)
-            .bind(timestamp_from)
-            .bind(timestamp_to),
+            paramsv![
+                disable_chat_id,
+                chat_id,
+                disable_contact_id,
+                contact_id as i32,
+                timestamp_from,
+                timestamp_to,
+            ],
+            |row| {
+                let msg_id = row.get(6)?;
+                let txt: String = row.get(9)?;
+                let marker = if msg_id != 0 && is_marker(&txt) {
+                    Some(txt)
+                } else {
+                    None
+                };
+                let loc = Location {
+                    location_id: row.get(0)?,
+                    latitude: row.get(1)?,
+                    longitude: row.get(2)?,
+                    accuracy: row.get(3)?,
+                    timestamp: row.get(4)?,
+                    independent: row.get(5)?,
+                    msg_id,
+                    contact_id: row.get(7)?,
+                    chat_id: row.get(8)?,
+                    marker,
+                };
+                Ok(loc)
+            },
+            |locations| {
+                let mut ret = Vec::new();
+
+                for location in locations {
+                    ret.push(location?);
+                }
+                Ok(ret)
+            },
         )
-        .await?
-        .map(|row| {
-            let row = row?;
-            let msg_id = row.try_get(6)?;
-            let txt: String = row.try_get(9)?;
-            let marker = if msg_id != 0 && is_marker(&txt) {
-                Some(txt)
-            } else {
-                None
-            };
-            let loc = Location {
-                location_id: row.try_get(0)?,
-                latitude: row.try_get(1)?,
-                longitude: row.try_get(2)?,
-                accuracy: row.try_get(3)?,
-                timestamp: row.try_get(4)?,
-                independent: row.try_get(5)?,
-                msg_id,
-                contact_id: row.try_get(7)?,
-                chat_id: row.try_get(8)?,
-                marker,
-            };
-            Ok(loc)
-        })
-        .collect::<sqlx::Result<_>>()
         .await?;
     Ok(list)
 }
@@ -403,7 +402,7 @@ fn is_marker(txt: &str) -> bool {
 pub async fn delete_all(context: &Context) -> Result<(), Error> {
     context
         .sql
-        .execute(sqlx::query("DELETE FROM locations;"))
+        .execute("DELETE FROM locations;", paramsv![])
         .await?;
     context.emit_event(EventType::LocationChanged(None));
     Ok(())
@@ -417,65 +416,70 @@ pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)
         .await?
         .unwrap_or_default();
 
-    let (locations_send_begin, locations_send_until, locations_last_sent) = {
-        let row = context.sql.fetch_one(
-            sqlx::query(
-                "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;"
-            )
-                .bind(chat_id)
-        ).await?;
+    let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
+        "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
+        paramsv![chat_id], |row| {
+            let send_begin: i64 = row.get(0)?;
+            let send_until: i64 = row.get(1)?;
+            let last_sent: i64 = row.get(2)?;
 
-        let send_begin: i64 = row.try_get(0)?;
-        let send_until: i64 = row.try_get(1)?;
-        let last_sent: i64 = row.try_get(2)?;
-
-        (send_begin, send_until, last_sent)
-    };
+            Ok((send_begin, send_until, last_sent))
+        })
+        .await?;
 
     let now = time();
     let mut location_count = 0;
     let mut ret = String::new();
     if locations_send_begin != 0 && now <= locations_send_until {
         ret += &format!(
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
+            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
+            <kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
             self_addr,
         );
 
-        let mut rows = context.sql.fetch(
-            sqlx::query(
+        context
+            .sql
+            .query_map(
                 "SELECT id, latitude, longitude, accuracy, timestamp \
              FROM locations  WHERE from_id=? \
              AND timestamp>=? \
-             AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
+             AND (timestamp>=? OR \
+                  timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
              AND independent=0 \
              GROUP BY timestamp \
-             ORDER BY timestamp;"
+             ORDER BY timestamp;",
+                paramsv![
+                    DC_CONTACT_ID_SELF,
+                    locations_send_begin,
+                    locations_last_sent,
+                    DC_CONTACT_ID_SELF
+                ],
+                |row| {
+                    let location_id: i32 = row.get(0)?;
+                    let latitude: f64 = row.get(1)?;
+                    let longitude: f64 = row.get(2)?;
+                    let accuracy: f64 = row.get(3)?;
+                    let timestamp = get_kml_timestamp(row.get(4)?);
+
+                    Ok((location_id, latitude, longitude, accuracy, timestamp))
+                },
+                |rows| {
+                    for row in rows {
+                        let (location_id, latitude, longitude, accuracy, timestamp) = row?;
+                        ret += &format!(
+                            "<Placemark>\
+                <Timestamp><when>{}</when></Timestamp>\
+                <Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point>\
+                </Placemark>\n",
+                            timestamp, accuracy, longitude, latitude
+                        );
+                        location_count += 1;
+                        last_added_location_id = location_id as u32;
+                    }
+                    Ok(())
+                },
             )
-                .bind(DC_CONTACT_ID_SELF)
-                .bind(locations_send_begin)
-                .bind(locations_last_sent)
-                .bind(DC_CONTACT_ID_SELF)
-        ).await?;
-
-        while let Some(row) = rows.next().await {
-            let row = row?;
-            let location_id: u32 = row.try_get(0)?;
-            let latitude: f64 = row.try_get(1)?;
-            let longitude: f64 = row.try_get(2)?;
-            let accuracy: f64 = row.try_get(3)?;
-            let timestamp = get_kml_timestamp(row.try_get(4)?);
-
-            ret += &format!(
-                "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
-                timestamp,
-                accuracy,
-                longitude,
-                latitude
-            );
-            location_count += 1;
-            last_added_location_id = location_id;
-        }
-
+            .await?;
         ret += "</Document>\n</kml>";
     }
 
@@ -516,9 +520,8 @@ pub async fn set_kml_sent_timestamp(
     context
         .sql
         .execute(
-            sqlx::query("UPDATE chats SET locations_last_sent=? WHERE id=?;")
-                .bind(timestamp)
-                .bind(chat_id),
+            "UPDATE chats SET locations_last_sent=? WHERE id=?;",
+            paramsv![timestamp, chat_id],
         )
         .await?;
     Ok(())
@@ -532,9 +535,8 @@ pub async fn set_msg_location_id(
     context
         .sql
         .execute(
-            sqlx::query("UPDATE msgs SET location_id=? WHERE id=?;")
-                .bind(location_id)
-                .bind(msg_id),
+            "UPDATE msgs SET location_id=? WHERE id=?;",
+            paramsv![location_id, msg_id],
         )
         .await?;
 
@@ -553,7 +555,6 @@ pub async fn save(
     let mut newest_timestamp = 0;
     let mut newest_location_id = 0;
 
-    let stmt_test = "SELECT COUNT(*) FROM locations WHERE timestamp=? AND from_id=?";
     let stmt_insert = "INSERT INTO locations\
              (timestamp, from_id, chat_id, latitude, longitude, accuracy, independent) \
              VALUES (?,?,?,?,?,?,?);";
@@ -566,37 +567,31 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        let exists = context
-            .sql
-            .exists(sqlx::query(stmt_test).bind(timestamp).bind(contact_id))
-            .await?;
+
+        let conn = context.sql.get_conn().await?;
+        let mut stmt_test =
+            conn.prepare_cached("SELECT id FROM locations WHERE timestamp=? AND from_id=?")?;
+        let mut stmt_insert = conn.prepare_cached(stmt_insert)?;
+
+        let exists = stmt_test.exists(paramsv![timestamp, contact_id as i32])?;
+
         if independent || !exists {
-            context
-                .sql
-                .execute(
-                    sqlx::query(stmt_insert)
-                        .bind(timestamp)
-                        .bind(contact_id)
-                        .bind(chat_id)
-                        .bind(latitude)
-                        .bind(longitude)
-                        .bind(accuracy)
-                        .bind(independent),
-                )
-                .await?;
+            stmt_insert.execute(paramsv![
+                timestamp,
+                contact_id as i32,
+                chat_id,
+                latitude,
+                longitude,
+                accuracy,
+                independent,
+            ])?;
 
             if timestamp > newest_timestamp {
+                // okay to drop, as we use cached prepared statements
+                drop(stmt_test);
+                drop(stmt_insert);
                 newest_timestamp = timestamp;
-                newest_location_id = context
-                    .sql
-                    .get_rowid2(
-                        "locations",
-                        "timestamp",
-                        timestamp,
-                        "from_id",
-                        contact_id as i64,
-                    )
-                    .await?;
+                newest_location_id = conn.last_insert_rowid();
             }
         }
     }
@@ -614,21 +609,15 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
 
     let rows = context
         .sql
-        .fetch(
-            sqlx::query(
-                "SELECT id, locations_send_begin, locations_last_sent \
+        .query_map(
+            "SELECT id, locations_send_begin, locations_last_sent \
          FROM chats \
          WHERE locations_send_until>?;",
-            )
-            .bind(now),
-        )
-        .await
-        .map(|rows| {
-            rows.map(|row| -> sqlx::Result<Option<_>> {
-                let row = row?;
-                let chat_id: ChatId = row.try_get(0)?;
-                let locations_send_begin: i64 = row.try_get(1)?;
-                let locations_last_sent: i64 = row.try_get(2)?;
+            paramsv![now],
+            |row| {
+                let chat_id: ChatId = row.get(0)?;
+                let locations_send_begin: i64 = row.get(1)?;
+                let locations_last_sent: i64 = row.get(2)?;
                 continue_streaming = true;
 
                 // be a bit tolerant as the timer may not align exactly with time(NULL)
@@ -637,53 +626,57 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
                 } else {
                     Ok(Some((chat_id, locations_send_begin, locations_last_sent)))
                 }
-            })
-            .filter_map(|v| v.transpose())
-        });
+            },
+            |rows| {
+                rows.filter_map(|v| v.transpose())
+                    .collect::<Result<Vec<_>, _>>()
+                    .map_err(Into::into)
+            },
+        )
+        .await;
 
-    let stmt = "SELECT COUNT(*) \
-         FROM locations \
-         WHERE from_id=? \
-         AND timestamp>=? \
-         AND timestamp>? \
-         AND independent=0 \
-         ORDER BY timestamp;";
-
-    if let Ok(mut rows) = rows {
+    if let Ok(rows) = rows {
         let mut msgs = Vec::new();
-        while let Some(row) = rows.next().await {
-            let (chat_id, locations_send_begin, locations_last_sent) = match row {
-                Ok(row) => row,
-                Err(_) => break,
-            };
-            let exists = context
-                .sql
-                .exists(
-                    sqlx::query(stmt)
-                        .bind(DC_CONTACT_ID_SELF)
-                        .bind(locations_send_begin)
-                        .bind(locations_last_sent),
-                )
-                .await
-                .unwrap_or_default(); // TODO: better error handling
 
-            if !exists {
-                // if there is no new location, there's nothing to send.
-                // however, maybe we want to bypass this test eg. 15 minutes
-            } else {
-                // pending locations are attached automatically to every message,
-                // so also to this empty text message.
-                // DC_CMD_LOCATION is only needed to create a nicer subject.
-                //
-                // for optimisation and to avoid flooding the sending queue,
-                // we could sending these messages only if we're really online.
-                // the easiest way to determine this, is to check for an empty message queue.
-                // (might not be 100%, however, as positions are sent combined later
-                // and dc_set_location() is typically called periodically, this is ok)
-                let mut msg = Message::new(Viewtype::Text);
-                msg.hidden = true;
-                msg.param.set_cmd(SystemMessage::LocationOnly);
-                msgs.push((chat_id, msg));
+        {
+            let conn = job_try!(context.sql.get_conn().await);
+
+            let mut stmt_locations = job_try!(conn.prepare_cached(
+                "SELECT id \
+     FROM locations \
+     WHERE from_id=? \
+     AND timestamp>=? \
+     AND timestamp>? \
+     AND independent=0 \
+             ORDER BY timestamp;",
+            ));
+
+            for (chat_id, locations_send_begin, locations_last_sent) in &rows {
+                if !stmt_locations
+                    .exists(paramsv![
+                        DC_CONTACT_ID_SELF,
+                        *locations_send_begin,
+                        *locations_last_sent,
+                    ])
+                    .unwrap_or_default()
+                {
+                    // if there is no new location, there's nothing to send.
+                    // however, maybe we want to bypass this test eg. 15 minutes
+                } else {
+                    // pending locations are attached automatically to every message,
+                    // so also to this empty text message.
+                    // DC_CMD_LOCATION is only needed to create a nicer subject.
+                    //
+                    // for optimisation and to avoid flooding the sending queue,
+                    // we could sending these messages only if we're really online.
+                    // the easiest way to determine this, is to check for an empty message queue.
+                    // (might not be 100%, however, as positions are sent combined later
+                    // and dc_set_location() is typically called periodically, this is ok)
+                    let mut msg = Message::new(Viewtype::Text);
+                    msg.hidden = true;
+                    msg.param.set_cmd(SystemMessage::LocationOnly);
+                    msgs.push((*chat_id, msg));
+                }
             }
         }
 
@@ -711,16 +704,16 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
     let chat_id = ChatId::new(job.foreign_id);
 
-    let (send_begin, send_until) = job_try!(context
-        .sql
-        .fetch_one(
-            sqlx::query(
+    let (send_begin, send_until) = job_try!(
+        context
+            .sql
+            .query_row(
                 "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
+                paramsv![chat_id],
+                |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
             )
-            .bind(chat_id)
-        )
-        .await
-        .and_then(|row| { Ok((row.try_get::<i64, _>(0)?, row.try_get::<i64, _>(1)?)) }));
+            .await
+    );
 
     if !(send_begin != 0 && time() <= send_until) {
         // still streaming -
@@ -732,12 +725,10 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 context
                     .sql
                     .execute(
-                        sqlx::query(
-                            "UPDATE chats \
+                        "UPDATE chats \
                              SET locations_send_begin=0, locations_send_until=0 \
-                             WHERE id=?"
-                        )
-                        .bind(chat_id)
+ WHERE id=?",
+                        paramsv![chat_id],
                     )
                     .await
             );

src/log.rs

@@ -126,7 +126,7 @@ where
     }
 }
 
-impl<T: Default, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
+impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
     #[track_caller]
     fn log_err_inner(self, context: &Context, msg: Option<&str>) -> Result<T, E> {
         if let Err(e) = &self {

src/login_param.rs

@@ -5,6 +5,7 @@ use std::fmt;
 
 use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
+use anyhow::Result;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
@@ -30,7 +31,7 @@ impl Default for CertificateChecks {
 }
 
 /// Login parameters for a single server, either IMAP or SMTP
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub struct ServerLoginParam {
     pub server: String,
     pub user: String,
@@ -43,7 +44,7 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
-#[derive(Default, Debug, Clone)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub struct LoginParam {
     pub addr: String,
     pub imap: ServerLoginParam,
@@ -54,10 +55,7 @@ pub struct LoginParam {
 
 impl LoginParam {
     /// Read the login parameters from the database.
-    pub async fn from_database(
-        context: &Context,
-        prefix: impl AsRef<str>,
-    ) -> crate::sql::Result<Self> {
+    pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Result<Self> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -156,11 +154,7 @@ impl LoginParam {
     }
 
     /// Save this loginparam to the database.
-    pub async fn save_to_database(
-        &self,
-        context: &Context,
-        prefix: impl AsRef<str>,
-    ) -> crate::sql::Result<()> {
+    pub async fn save_to_database(&self, context: &Context, prefix: impl AsRef<str>) -> Result<()> {
         let prefix = prefix.as_ref();
         let sql = &context.sql;
 
@@ -304,6 +298,8 @@ pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
 mod tests {
     use super::*;
 
+    use crate::test_utils::TestContext;
+
     #[test]
     fn test_certificate_checks_display() {
         use std::string::ToString;
@@ -313,4 +309,37 @@ mod tests {
             CertificateChecks::AcceptInvalidCertificates.to_string()
         );
     }
+
+    #[async_std::test]
+    async fn test_save_load_login_param() -> Result<()> {
+        let t = TestContext::new().await;
+
+        let param = LoginParam {
+            addr: "alice@example.com".to_string(),
+            imap: ServerLoginParam {
+                server: "imap.example.com".to_string(),
+                user: "alice".to_string(),
+                password: "foo".to_string(),
+                port: 123,
+                security: Socket::Starttls,
+                certificate_checks: CertificateChecks::Strict,
+            },
+            smtp: ServerLoginParam {
+                server: "smtp.example.com".to_string(),
+                user: "alice@example.com".to_string(),
+                password: "bar".to_string(),
+                port: 456,
+                security: Socket::Ssl,
+                certificate_checks: CertificateChecks::AcceptInvalidCertificates,
+            },
+            server_flags: 0,
+            provider: get_provider_by_id("example.com"),
+        };
+
+        param.save_to_database(&t, "foobar_").await?;
+        let loaded = LoginParam::from_database(&t, "foobar_").await?;
+
+        assert_eq!(param, loaded);
+        Ok(())
+    }
 }

src/lot.rs

@@ -1,3 +1,5 @@
+use deltachat_derive::{FromSql, ToSql};
+
 use crate::key::Fingerprint;
 
 /// An object containing a set of values.
@@ -20,7 +22,9 @@ pub struct Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum Meaning {
     None = 0,
     Text1Draft = 1,
@@ -64,8 +68,10 @@ impl Lot {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(u32)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 pub enum LotState {
     // Default
     Undefined = 0,

src/mimefactory.rs

@@ -1,10 +1,8 @@
 use std::convert::TryInto;
 
-use anyhow::{bail, ensure, format_err, Error};
-use async_std::prelude::*;
+use anyhow::{bail, ensure, format_err, Result};
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
-use sqlx::Row;
 
 use crate::blob::BlobObject;
 use crate::chat::{self, Chat};
@@ -85,12 +83,39 @@ pub struct RenderedEmail {
     pub subject: String,
 }
 
+#[derive(Debug, Clone, Default)]
+struct MessageHeaders {
+    /// Opportunistically protected headers.
+    ///
+    /// These headers are placed into encrypted part *if* the message is encrypted. Place headers
+    /// which are not needed before decryption (e.g. Chat-Group-Name) or are not interesting if the
+    /// message cannot be decrypted (e.g. Chat-Disposition-Notification-To) here.
+    ///
+    /// If the message is not encrypted, these headers are placed into IMF header section, so make
+    /// sure that the message will be encrypted if you place any sensitive information here.
+    pub protected: Vec<Header>,
+
+    /// Headers that must go into IMF header section.
+    ///
+    /// These are standard headers such as Date, In-Reply-To, References, which cannot be placed
+    /// anywhere else according to the standard. Placing headers here also allows them to be fetched
+    /// individually over IMAP without downloading the message body. This is why Chat-Version is
+    /// placed here.
+    pub unprotected: Vec<Header>,
+
+    /// Headers that MUST NOT go into IMF header section.
+    ///
+    /// These are large headers which may hit the header section size limit on the server, such as
+    /// Chat-User-Avatar with a base64-encoded image inside.
+    pub hidden: Vec<Header>,
+}
+
 impl<'a> MimeFactory<'a> {
     pub async fn from_msg(
         context: &Context,
         msg: &'a Message,
         attach_selfavatar: bool,
-    ) -> Result<MimeFactory<'a>, Error> {
+    ) -> Result<MimeFactory<'a>> {
         let chat = Chat::load_from_db(context, msg.chat_id).await?;
 
         let from_addr = context
@@ -115,42 +140,51 @@ impl<'a> MimeFactory<'a> {
         if chat.is_self_talk() {
             recipients.push((from_displayname.to_string(), from_addr.to_string()));
         } else {
-            let mut rows = context
+            context
                 .sql
-                .fetch(
-                    sqlx::query(
-                        "SELECT c.authname, c.addr  \
+                .query_map(
+                    "SELECT c.authname, c.addr  \
                  FROM chats_contacts cc  \
                  LEFT JOIN contacts c ON cc.contact_id=c.id  \
                  WHERE cc.chat_id=? AND cc.contact_id>9;",
-                    )
-                    .bind(msg.chat_id),
+                    paramsv![msg.chat_id],
+                    |row| {
+                        let authname: String = row.get(0)?;
+                        let addr: String = row.get(1)?;
+                        Ok((authname, addr))
+                    },
+                    |rows| {
+                        for row in rows {
+                            let (authname, addr) = row?;
+                            if !recipients_contain_addr(&recipients, &addr) {
+                                recipients.push((authname, addr));
+                            }
+                        }
+                        Ok(())
+                    },
                 )
                 .await?;
-            while let Some(row) = rows.next().await {
-                let row = row?;
-                let authname: String = row.try_get(0)?;
-                let addr: String = row.try_get(1)?;
-                if !recipients_contain_addr(&recipients, &addr) {
-                    recipients.push((authname, addr));
-                }
-            }
 
             if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await? {
                 req_mdn = true;
             }
         }
-        let row = context
+        let (in_reply_to, references) = context
             .sql
-            .fetch_one(
-                sqlx::query("SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?")
-                    .bind(msg.id),
+            .query_row(
+                "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
+                paramsv![msg.id],
+                |row| {
+                    let in_reply_to: String = row.get(0)?;
+                    let references: String = row.get(1)?;
+
+                    Ok((
+                        render_rfc724_mid_list(&in_reply_to),
+                        render_rfc724_mid_list(&references),
+                    ))
+                },
             )
             .await?;
-        let (in_reply_to, references) = (
-            render_rfc724_mid_list(row.try_get(0)?),
-            render_rfc724_mid_list(row.try_get(1)?),
-        );
 
         let default_str = stock_str::status_line(context).await;
         let factory = MimeFactory {
@@ -178,7 +212,7 @@ impl<'a> MimeFactory<'a> {
         context: &Context,
         msg: &'a Message,
         additional_msg_ids: Vec<String>,
-    ) -> Result<MimeFactory<'a>, Error> {
+    ) -> Result<MimeFactory<'a>> {
         ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id).await?;
@@ -222,7 +256,7 @@ impl<'a> MimeFactory<'a> {
     async fn peerstates_for_recipients(
         &self,
         context: &Context,
-    ) -> Result<Vec<(Option<Peerstate>, &str)>, Error> {
+    ) -> Result<Vec<(Option<Peerstate>, &str)>> {
         let self_addr = context
             .get_config(Config::ConfiguredAddr)
             .await?
@@ -302,7 +336,7 @@ impl<'a> MimeFactory<'a> {
         }
     }
 
-    async fn should_do_gossip(&self, context: &Context) -> Result<bool, Error> {
+    async fn should_do_gossip(&self, context: &Context) -> Result<bool> {
         match &self.loaded {
             Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
@@ -401,15 +435,8 @@ impl<'a> MimeFactory<'a> {
             .collect()
     }
 
-    pub async fn render(mut self, context: &Context) -> Result<RenderedEmail, Error> {
-        // Headers that are encrypted
-        // - Chat-*, except Chat-Version
-        // - Secure-Join*
-        // - Subject
-        let mut protected_headers: Vec<Header> = Vec::new();
-
-        // All other headers
-        let mut unprotected_headers: Vec<Header> = Vec::new();
+    pub async fn render(mut self, context: &Context) -> Result<RenderedEmail> {
+        let mut headers: MessageHeaders = Default::default();
 
         let from = Address::new_mailbox_with_name(
             self.from_displayname.to_string(),
@@ -432,14 +459,20 @@ impl<'a> MimeFactory<'a> {
             to.push(from.clone());
         }
 
-        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+        headers
+            .unprotected
+            .push(Header::new("MIME-Version".into(), "1.0".into()));
 
         if !self.references.is_empty() {
-            unprotected_headers.push(Header::new("References".into(), self.references.clone()));
+            headers
+                .unprotected
+                .push(Header::new("References".into(), self.references.clone()));
         }
 
         if !self.in_reply_to.is_empty() {
-            unprotected_headers.push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
+            headers
+                .unprotected
+                .push(Header::new("In-Reply-To".into(), self.in_reply_to.clone()));
         }
 
         let date = chrono::Utc
@@ -447,12 +480,14 @@ impl<'a> MimeFactory<'a> {
             .unwrap()
             .to_rfc2822();
 
-        unprotected_headers.push(Header::new("Date".into(), date));
+        headers.unprotected.push(Header::new("Date".into(), date));
 
-        unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
+        headers
+            .unprotected
+            .push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::Mdn { .. } = self.loaded {
-            unprotected_headers.push(Header::new(
+            headers.unprotected.push(Header::new(
                 "Auto-Submitted".to_string(),
                 "auto-replied".to_string(),
             ));
@@ -462,7 +497,7 @@ impl<'a> MimeFactory<'a> {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
             // eg. are just freetext and/or do not follow any standard.
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Disposition-Notification-To".into(),
                 self.from_addr.clone(),
             ));
@@ -490,10 +525,14 @@ impl<'a> MimeFactory<'a> {
         if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
-            unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
+            headers
+                .unprotected
+                .push(Header::new("Autocrypt".into(), aheader));
         }
 
-        protected_headers.push(Header::new("Subject".into(), encoded_subject));
+        headers
+            .protected
+            .push(Header::new("Subject".into(), encoded_subject));
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
@@ -502,23 +541,28 @@ impl<'a> MimeFactory<'a> {
 
         let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(context).await?;
         if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Ephemeral-Timer".to_string(),
                 duration.to_string(),
             ));
         }
 
-        unprotected_headers.push(Header::new(
+        headers.unprotected.push(Header::new(
             "Message-ID".into(),
             render_rfc724_mid(&rfc724_mid),
         ));
 
-        unprotected_headers.push(Header::new_with_value("To".into(), to).unwrap());
-        unprotected_headers.push(Header::new_with_value("From".into(), vec![from]).unwrap());
+        headers
+            .unprotected
+            .push(Header::new_with_value("To".into(), to).unwrap());
+        headers
+            .unprotected
+            .push(Header::new_with_value("From".into(), vec![from]).unwrap());
         if let Some(sender_displayname) = &self.sender_displayname {
             let sender =
                 Address::new_mailbox_with_name(sender_displayname.clone(), self.from_addr.clone());
-            unprotected_headers
+            headers
+                .unprotected
                 .push(Header::new_with_value("Sender".into(), vec![sender]).unwrap());
         }
 
@@ -526,13 +570,8 @@ impl<'a> MimeFactory<'a> {
 
         let (main_part, parts) = match self.loaded {
             Loaded::Message { .. } => {
-                self.render_message(
-                    context,
-                    &mut protected_headers,
-                    &mut unprotected_headers,
-                    &grpimage,
-                )
-                .await?
+                self.render_message(context, &mut headers, &grpimage)
+                    .await?
             }
             Loaded::Mdn { .. } => (self.render_mdn(context).await?, Vec::new()),
         };
@@ -555,12 +594,19 @@ impl<'a> MimeFactory<'a> {
             )
         };
 
-        // Store protected headers in the inner message.
-        let mut message = protected_headers
-            .into_iter()
-            .fold(message, |message, header| message.header(header));
-
         let outer_message = if is_encrypted {
+            // Store protected headers in the inner message.
+            let message = headers
+                .protected
+                .into_iter()
+                .fold(message, |message, header| message.header(header));
+
+            // Add hidden headers to encrypted payload.
+            let mut message = headers
+                .hidden
+                .into_iter()
+                .fold(message, |message, header| message.header(header));
+
             // Add gossip headers in chats with multiple recipients
             if peerstates.len() > 1 && self.should_do_gossip(context).await? {
                 for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) {
@@ -594,11 +640,6 @@ impl<'a> MimeFactory<'a> {
                 "multipart/encrypted; protocol=\"application/pgp-encrypted\"".to_string(),
             ));
 
-            // Store the unprotected headers on the outer message.
-            let outer_message = unprotected_headers
-                .into_iter()
-                .fold(outer_message, |message, header| message.header(header));
-
             if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
                 info!(context, "mimefactory: outgoing message mime:");
                 let raw_message = message.clone().build().as_string();
@@ -633,11 +674,33 @@ impl<'a> MimeFactory<'a> {
                 )
                 .header(("Subject".to_string(), "...".to_string()))
         } else {
-            unprotected_headers
+            let message = if headers.hidden.is_empty() {
+                message
+            } else {
+                // Store hidden headers in the inner unencrypted message.
+                let message = headers
+                    .hidden
+                    .into_iter()
+                    .fold(message, |message, header| message.header(header));
+
+                PartBuilder::new()
+                    .message_type(MimeMultipartType::Mixed)
+                    .child(message.build())
+            };
+
+            // Store protected headers in the outer message.
+            headers
+                .protected
                 .into_iter()
                 .fold(message, |message, header| message.header(header))
         };
 
+        // Store the unprotected headers on the outer message.
+        let outer_message = headers
+            .unprotected
+            .into_iter()
+            .fold(outer_message, |message, header| message.header(header));
+
         let MimeFactory {
             last_added_location_id,
             ..
@@ -673,7 +736,7 @@ impl<'a> MimeFactory<'a> {
         Some(part)
     }
 
-    async fn get_location_kml_part(&mut self, context: &Context) -> Result<PartBuilder, Error> {
+    async fn get_location_kml_part(&mut self, context: &Context) -> Result<PartBuilder> {
         let (kml_content, last_added_location_id) =
             location::get_kml(context, self.msg.chat_id).await?;
         let part = PartBuilder::new()
@@ -698,10 +761,9 @@ impl<'a> MimeFactory<'a> {
     async fn render_message(
         &mut self,
         context: &Context,
-        protected_headers: &mut Vec<Header>,
-        unprotected_headers: &mut Vec<Header>,
+        headers: &mut MessageHeaders,
         grpimage: &Option<String>,
-    ) -> Result<(PartBuilder, Vec<PartBuilder>), Error> {
+    ) -> Result<(PartBuilder, Vec<PartBuilder>)> {
         let chat = match &self.loaded {
             Loaded::Message { chat } => chat,
             Loaded::Mdn { .. } => bail!("Attempt to render MDN as a message"),
@@ -711,20 +773,26 @@ impl<'a> MimeFactory<'a> {
         let mut meta_part = None;
 
         if chat.is_protected() {
-            protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
+            headers
+                .protected
+                .push(Header::new("Chat-Verified".to_string(), "1".to_string()));
         }
 
         if chat.typ == Chattype::Group {
-            protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
-            protected_headers.push(Header::new("Chat-Group-Name".into(), encoded));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-Name".into(), encoded));
 
             match command {
                 SystemMessage::MemberRemovedFromGroup => {
                     let email_to_remove = self.msg.param.get(Param::Arg).unwrap_or_default();
                     if !email_to_remove.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Member-Removed".into(),
                             email_to_remove.into(),
                         ));
@@ -733,7 +801,7 @@ impl<'a> MimeFactory<'a> {
                 SystemMessage::MemberAddedToGroup => {
                     let email_to_add = self.msg.param.get(Param::Arg).unwrap_or_default();
                     if !email_to_add.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Member-Added".into(),
                             email_to_add.into(),
                         ));
@@ -746,7 +814,7 @@ impl<'a> MimeFactory<'a> {
                             "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>",
                             "vg-member-added",
                         );
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Secure-Join".to_string(),
                             "vg-member-added".to_string(),
                         ));
@@ -754,18 +822,18 @@ impl<'a> MimeFactory<'a> {
                 }
                 SystemMessage::GroupNameChanged => {
                     let old_name = self.msg.param.get(Param::Arg).unwrap_or_default();
-                    protected_headers.push(Header::new(
+                    headers.protected.push(Header::new(
                         "Chat-Group-Name-Changed".into(),
                         maybe_encode_words(old_name),
                     ));
                 }
                 SystemMessage::GroupImageChanged => {
-                    protected_headers.push(Header::new(
+                    headers.protected.push(Header::new(
                         "Chat-Content".to_string(),
                         "group-avatar-changed".to_string(),
                     ));
                     if grpimage.is_none() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Chat-Group-Avatar".to_string(),
                             "0".to_string(),
                         ));
@@ -777,13 +845,13 @@ impl<'a> MimeFactory<'a> {
 
         match command {
             SystemMessage::LocationStreamingEnabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".into(),
                     "location-streaming-enabled".into(),
                 ));
             }
             SystemMessage::EphemeralTimerChanged => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "ephemeral-timer-changed".to_string(),
                 ));
@@ -797,13 +865,14 @@ impl<'a> MimeFactory<'a> {
                 // Adding this header without encryption leaks some
                 // information about the message contents, but it can
                 // already be easily guessed from message timing and size.
-                unprotected_headers.push(Header::new(
+                headers.unprotected.push(Header::new(
                     "Auto-Submitted".to_string(),
                     "auto-generated".to_string(),
                 ));
             }
             SystemMessage::AutocryptSetupMessage => {
-                unprotected_headers
+                headers
+                    .unprotected
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
 
                 placeholdertext = Some(stock_str::ac_setup_msg_body(context).await);
@@ -816,11 +885,13 @@ impl<'a> MimeFactory<'a> {
                         context,
                         "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>", step,
                     );
-                    protected_headers.push(Header::new("Secure-Join".into(), step.into()));
+                    headers
+                        .protected
+                        .push(Header::new("Secure-Join".into(), step.into()));
 
                     let param2 = msg.param.get(Param::Arg2).unwrap_or_default();
                     if !param2.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             if step == "vg-request-with-auth" || step == "vc-request-with-auth" {
                                 "Secure-Join-Auth".into()
                             } else {
@@ -832,24 +903,26 @@ impl<'a> MimeFactory<'a> {
 
                     let fingerprint = msg.param.get(Param::Arg3).unwrap_or_default();
                     if !fingerprint.is_empty() {
-                        protected_headers.push(Header::new(
+                        headers.protected.push(Header::new(
                             "Secure-Join-Fingerprint".into(),
                             fingerprint.into(),
                         ));
                     }
                     if let Some(id) = msg.param.get(Param::Arg4) {
-                        protected_headers.push(Header::new("Secure-Join-Group".into(), id.into()));
+                        headers
+                            .protected
+                            .push(Header::new("Secure-Join-Group".into(), id.into()));
                     };
                 }
             }
             SystemMessage::ChatProtectionEnabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "protection-enabled".to_string(),
                 ));
             }
             SystemMessage::ChatProtectionDisabled => {
-                protected_headers.push(Header::new(
+                headers.protected.push(Header::new(
                     "Chat-Content".to_string(),
                     "protection-disabled".to_string(),
                 ));
@@ -867,17 +940,21 @@ impl<'a> MimeFactory<'a> {
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
             meta_part = Some(mail);
-            protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            headers
+                .protected
+                .push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
-            protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+            headers
+                .protected
+                .push(Header::new("Chat-Content".into(), "sticker".into()));
         } else if self.msg.viewtype == Viewtype::VideochatInvitation {
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Content".into(),
                 "videochat-invitation".into(),
             ));
-            protected_headers.push(Header::new(
+            headers.protected.push(Header::new(
                 "Chat-Webrtc-Room".into(),
                 self.msg
                     .param
@@ -892,12 +969,16 @@ impl<'a> MimeFactory<'a> {
             || self.msg.viewtype == Viewtype::Video
         {
             if self.msg.viewtype == Viewtype::Voice {
-                protected_headers.push(Header::new("Chat-Voice-Message".into(), "1".into()));
+                headers
+                    .protected
+                    .push(Header::new("Chat-Voice-Message".into(), "1".into()));
             }
             let duration_ms = self.msg.param.get_int(Param::Duration).unwrap_or_default();
             if duration_ms > 0 {
                 let dur = duration_ms.to_string();
-                protected_headers.push(Header::new("Chat-Duration".into(), dur));
+                headers
+                    .protected
+                    .push(Header::new("Chat-Duration".into(), dur));
             }
         }
 
@@ -977,7 +1058,7 @@ impl<'a> MimeFactory<'a> {
 
         // add attachment part
         if chat::msgtype_has_file(self.msg.viewtype) {
-            if !is_file_size_okay(context, self.msg).await {
+            if !is_file_size_okay(context, self.msg).await? {
                 bail!(
                     "Message exceeds the recommended {} MB.",
                     RECOMMENDED_FILE_SIZE / 1_000_000,
@@ -1008,13 +1089,15 @@ impl<'a> MimeFactory<'a> {
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
                 Some(path) => match build_selfavatar_file(context, &path) {
-                    Ok((part, filename)) => {
-                        parts.push(part);
-                        protected_headers.push(Header::new("Chat-User-Avatar".into(), filename))
-                    }
+                    Ok(avatar) => headers.hidden.push(Header::new(
+                        "Chat-User-Avatar".into(),
+                        format!("base64:{}", avatar),
+                    )),
                     Err(err) => warn!(context, "mimefactory: cannot attach selfavatar: {}", err),
                 },
-                None => protected_headers.push(Header::new("Chat-User-Avatar".into(), "0".into())),
+                None => headers
+                    .protected
+                    .push(Header::new("Chat-User-Avatar".into(), "0".into())),
             }
         }
 
@@ -1022,7 +1105,7 @@ impl<'a> MimeFactory<'a> {
     }
 
     /// Render an MDN
-    async fn render_mdn(&mut self, context: &Context) -> Result<PartBuilder, Error> {
+    async fn render_mdn(&mut self, context: &Context) -> Result<PartBuilder> {
         // RFC 6522, this also requires the `report-type` parameter which is equal
         // to the MIME subtype of the second body part of the multipart/report
         //
@@ -1060,10 +1143,13 @@ impl<'a> MimeFactory<'a> {
             self.msg.get_summarytext(context, 32).await
         };
         let p2 = stock_str::read_rcpt_mail_body(context, p1).await;
-        let message_text = format!("{}\r\n", p2);
+        let message_text = format!("{}\r\n", format_flowed(&p2));
         message = message.child(
             PartBuilder::new()
-                .content_type(&mime::TEXT_PLAIN_UTF_8)
+                .header((
+                    "Content-Type".to_string(),
+                    "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+                ))
                 .body(message_text)
                 .build(),
         );
@@ -1120,7 +1206,7 @@ async fn build_body_file(
     context: &Context,
     msg: &Message,
     base_name: &str,
-) -> Result<(PartBuilder, String), Error> {
+) -> Result<(PartBuilder, String)> {
     let blob = msg
         .param
         .get_blob(Param::File, context, true)
@@ -1194,29 +1280,11 @@ async fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String), Error> {
-    let blob = BlobObject::from_path(context, path)?;
-    let filename_to_send = match blob.suffix() {
-        Some(suffix) => format!("avatar.{}", suffix),
-        None => "avatar".to_string(),
-    };
-    let mimetype = match message::guess_msgtype_from_suffix(blob.as_rel_path()) {
-        Some(res) => res.1.parse()?,
-        None => mime::APPLICATION_OCTET_STREAM,
-    };
+fn build_selfavatar_file(context: &Context, path: &str) -> Result<String> {
+    let blob = BlobObject::from_path(context, path.as_ref())?;
     let body = std::fs::read(blob.to_abs_path())?;
     let encoded_body = wrapped_base64_encode(&body);
-
-    let part = PartBuilder::new()
-        .content_type(&mimetype)
-        .header((
-            "Content-Disposition",
-            format!("attachment; filename=\"{}\"", &filename_to_send),
-        ))
-        .header(("Content-Transfer-Encoding", "base64"))
-        .body(encoded_body);
-
-    Ok((part, filename_to_send))
+    Ok(encoded_body)
 }
 
 fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool {
@@ -1226,13 +1294,13 @@ fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool
         .any(|(_, cur)| cur.to_lowercase() == addr_lc)
 }
 
-async fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
-    match msg.param.get_path(Param::File, context).unwrap_or(None) {
+async fn is_file_size_okay(context: &Context, msg: &Message) -> Result<bool> {
+    match msg.param.get_path(Param::File, context)? {
         Some(path) => {
             let bytes = dc_get_filebytes(context, &path).await;
-            bytes <= UPPER_LIMIT_FILE_SIZE
+            Ok(bytes <= UPPER_LIMIT_FILE_SIZE)
         }
-        None => false,
+        None => Ok(false),
     }
 }
 
@@ -1263,8 +1331,8 @@ fn encode_words(word: &str) -> String {
     encoded_words::encode(word, None, encoded_words::EncodingFlag::Shortest, None)
 }
 
-fn needs_encoding(to_check: impl AsRef<str>) -> bool {
-    !to_check.as_ref().chars().all(|c| {
+fn needs_encoding(to_check: &str) -> bool {
+    !to_check.chars().all(|c| {
         c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' || c == '%'
     })
 }
@@ -1280,14 +1348,16 @@ fn maybe_encode_words(words: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat::ChatId;
+    use async_std::prelude::*;
 
+    use crate::chat::ChatId;
+    use crate::chatlist::Chatlist;
     use crate::contact::Origin;
     use crate::dc_receive_imf::dc_receive_imf;
     use crate::mimeparser::MimeMessage;
-    use crate::test_utils::TestContext;
-    use crate::{chatlist::Chatlist, test_utils::get_chat_msg};
+    use crate::test_utils::{get_chat_msg, TestContext};
 
+    use async_std::fs::File;
     use pretty_assertions::assert_eq;
 
     #[test]
@@ -1620,7 +1690,7 @@ mod tests {
                 .unwrap()
                 .0;
 
-        let chat_id = chat::create_by_contact_id(&t, contact_id).await.unwrap();
+        let chat_id = ChatId::create_for_contact(&t, contact_id).await.unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
         new_msg.set_text(Some("Hi".to_string()));
@@ -1838,4 +1908,59 @@ mod tests {
 
         assert!(!headers.lines().any(|l| l.trim().is_empty()));
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_unencrypted() -> anyhow::Result<()> {
+        // create chat with bob, set selfavatar
+        let t = TestContext::new_alice().await;
+        let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
+
+        let file = t.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&file).await?.write_all(bytes).await?;
+        t.set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+
+        // send message to bob: that should get multipart/mixed because of the avatar moved to inner header;
+        // make sure, `Subject:` stays in the outer header (imf header)
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("this is the text!".to_string()));
+
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(3, "\r\n\r\n");
+        let outer = payload.next().unwrap();
+        let inner = payload.next().unwrap();
+        let body = payload.next().unwrap();
+
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(outer.match_indices("Subject:").count(), 1);
+        assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+        assert_eq!(inner.match_indices("text/plain").count(), 1);
+        assert_eq!(inner.match_indices("Chat-User-Avatar:").count(), 1);
+        assert_eq!(inner.match_indices("Subject:").count(), 0);
+
+        assert_eq!(body.match_indices("this is the text!").count(), 1);
+
+        // if another message is sent, that one must not contain the avatar
+        // and no artificial multipart/mixed nesting
+        let payload = t.send_msg(chat.id, &mut msg).await.payload();
+        let mut payload = payload.splitn(2, "\r\n\r\n");
+        let outer = payload.next().unwrap();
+        let body = payload.next().unwrap();
+
+        assert_eq!(outer.match_indices("text/plain").count(), 1);
+        assert_eq!(outer.match_indices("Subject:").count(), 1);
+        assert_eq!(outer.match_indices("Autocrypt:").count(), 1);
+        assert_eq!(outer.match_indices("multipart/mixed").count(), 0);
+        assert_eq!(outer.match_indices("Chat-User-Avatar:").count(), 0);
+
+        assert_eq!(body.match_indices("this is the text!").count(), 1);
+        assert_eq!(body.match_indices("text/plain").count(), 0);
+        assert_eq!(body.match_indices("Chat-User-Avatar:").count(), 0);
+        assert_eq!(body.match_indices("Subject:").count(), 0);
+
+        Ok(())
+    }
 }

src/mimeparser.rs

@@ -4,6 +4,7 @@ use std::pin::Pin;
 
 use anyhow::{bail, Result};
 use charset::Charset;
+use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 use once_cell::sync::Lazy;
@@ -102,7 +103,9 @@ pub(crate) enum MailinglistType {
     None,
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql,
+)]
 #[repr(u32)]
 pub enum SystemMessage {
     Unknown = 0,
@@ -146,7 +149,7 @@ impl MimeMessage {
         let mut from = Default::default();
         let mut chat_disposition_notification_to = None;
 
-        // init known headers with what mailparse provided us
+        // Parse IMF headers.
         MimeMessage::merge_headers(
             context,
             &mut headers,
@@ -156,6 +159,21 @@ impl MimeMessage {
             &mail.headers,
         );
 
+        // Parse hidden headers.
+        let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+        if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" {
+            if let Some(part) = mail.subparts.first() {
+                for field in &part.headers {
+                    let key = field.get_key().to_lowercase();
+
+                    // For now only Chat-User-Avatar can be hidden.
+                    if !headers.contains_key(&key) && key == "chat-user-avatar" {
+                        headers.insert(key.to_string(), field.get_value());
+                    }
+                }
+            }
+        }
+
         // remove headers that are allowed _only_ in the encrypted part
         headers.remove("secure-join-fingerprint");
         headers.remove("chat-verified");
@@ -260,7 +278,7 @@ impl MimeMessage {
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
-        parser.parse_headers(context);
+        parser.parse_headers(context).await;
 
         if warn_empty_signature && parser.signatures.is_empty() {
             for part in parser.parts.iter_mut() {
@@ -307,13 +325,13 @@ impl MimeMessage {
     }
 
     /// Parses avatar action headers.
-    fn parse_avatar_headers(&mut self) {
+    async fn parse_avatar_headers(&mut self, context: &Context) {
         if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
-            self.group_avatar = self.avatar_action_from_header(header_value);
+            self.group_avatar = self.avatar_action_from_header(context, header_value).await;
         }
 
         if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
-            self.user_avatar = self.avatar_action_from_header(header_value);
+            self.user_avatar = self.avatar_action_from_header(context, header_value).await;
         }
     }
 
@@ -403,9 +421,9 @@ impl MimeMessage {
         }
     }
 
-    fn parse_headers(&mut self, context: &Context) {
+    async fn parse_headers(&mut self, context: &Context) {
         self.parse_system_message_headers(context);
-        self.parse_avatar_headers();
+        self.parse_avatar_headers(context).await;
         self.parse_videochat_headers();
         self.squash_attachment_parts();
 
@@ -482,10 +500,48 @@ impl MimeMessage {
         }
     }
 
-    fn avatar_action_from_header(&mut self, header_value: String) -> Option<AvatarAction> {
+    async fn avatar_action_from_header(
+        &mut self,
+        context: &Context,
+        header_value: String,
+    ) -> Option<AvatarAction> {
         if header_value == "0" {
             Some(AvatarAction::Delete)
+        } else if let Some(avatar) = header_value
+            .split_ascii_whitespace()
+            .collect::<String>()
+            .strip_prefix("base64:")
+            .map(base64::decode)
+        {
+            // Avatar sent directly in the header as base64.
+            if let Ok(decoded_data) = avatar {
+                let extension = if let Ok(format) = image::guess_format(&decoded_data) {
+                    if let Some(ext) = format.extensions_str().first() {
+                        format!(".{}", ext)
+                    } else {
+                        String::new()
+                    }
+                } else {
+                    String::new()
+                };
+                match BlobObject::create(context, &format!("avatar{}", extension), &decoded_data)
+                    .await
+                {
+                    Ok(blob) => Some(AvatarAction::Change(blob.as_name().to_string())),
+                    Err(err) => {
+                        warn!(
+                            context,
+                            "Could not save decoded avatar to blob file: {}", err
+                        );
+                        None
+                    }
+                }
+            } else {
+                None
+            }
         } else {
+            // Avatar sent in attachment, as previous versions of Delta Chat did.
+
             let mut i = 0;
             while let Some(part) = self.parts.get_mut(i) {
                 if let Some(part_filename) = &part.org_filename {
@@ -1249,7 +1305,8 @@ impl MimeMessage {
             context
                 .sql
                 .query_get_value(
-                    sqlx::query("SELECT timestamp FROM msgs WHERE rfc724_mid=?").bind(field),
+                    "SELECT timestamp FROM msgs WHERE rfc724_mid=?",
+                    paramsv![field],
                 )
                 .await?
         } else {
@@ -1528,7 +1585,7 @@ fn get_attachment_filename(
 /// Returned addresses are normalized and lowercased.
 pub(crate) fn get_recipients(headers: &[MailHeader]) -> Vec<SingleInfo> {
     get_all_addresses_from_header(headers, |header_key| {
-        header_key == "to" || header_key == "cc" || header_key == "bcc"
+        header_key == "to" || header_key == "cc"
     })
 }
 
@@ -1920,9 +1977,8 @@ mod tests {
             .ctx
             .sql
             .execute(
-                sqlx::query("INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)")
-                    .bind("Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org")
-                    .bind(timestamp),
+                "INSERT INTO msgs (rfc724_mid, timestamp) VALUES(?,?)",
+                paramsv!["Gr.beZgAF2Nn0-.oyaJOpeuT70@example.org", timestamp],
             )
             .await
             .expect("Failed to write to the database");

src/oauth2.rs

@@ -53,20 +53,20 @@ struct Response {
 
 pub async fn dc_get_oauth2_url(
     context: &Context,
-    addr: impl AsRef<str>,
-    redirect_uri: impl AsRef<str>,
+    addr: &str,
+    redirect_uri: &str,
 ) -> Option<String> {
     if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
-            .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri.as_ref()))
+            .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
             .await
             .is_err()
         {
             return None;
         }
-        let oauth2_url = replace_in_uri(&oauth2.get_code, "$CLIENT_ID", &oauth2.client_id);
-        let oauth2_url = replace_in_uri(&oauth2_url, "$REDIRECT_URI", redirect_uri.as_ref());
+        let oauth2_url = replace_in_uri(oauth2.get_code, "$CLIENT_ID", oauth2.client_id);
+        let oauth2_url = replace_in_uri(&oauth2_url, "$REDIRECT_URI", redirect_uri);
 
         Some(oauth2_url)
     } else {
@@ -76,8 +76,8 @@ pub async fn dc_get_oauth2_url(
 
 pub async fn dc_get_oauth2_access_token(
     context: &Context,
-    addr: impl AsRef<str>,
-    code: impl AsRef<str>,
+    addr: &str,
+    code: &str,
     regenerate: bool,
 ) -> Result<Option<String>> {
     if let Some(oauth2) = Oauth2::from_address(addr).await {
@@ -101,7 +101,7 @@ pub async fn dc_get_oauth2_access_token(
             .unwrap_or_else(|| "unset".into());
 
         let (redirect_uri, token_url, update_redirect_uri_on_success) =
-            if refresh_token.is_none() || refresh_token_for != code.as_ref() {
+            if refresh_token.is_none() || refresh_token_for != code {
                 info!(context, "Generate OAuth2 refresh_token and access_token...",);
                 (
                     context
@@ -145,7 +145,7 @@ pub async fn dc_get_oauth2_access_token(
             } else if value == "$REDIRECT_URI" {
                 value = &redirect_uri;
             } else if value == "$CODE" {
-                value = code.as_ref();
+                value = code;
             } else if value == "$REFRESH_TOKEN" && refresh_token.is_some() {
                 value = refresh_token.as_ref().unwrap();
             }
@@ -179,7 +179,7 @@ pub async fn dc_get_oauth2_access_token(
                 .await?;
             context
                 .sql
-                .set_raw_config("oauth2_refresh_token_for", Some(code.as_ref()))
+                .set_raw_config("oauth2_refresh_token_for", Some(code))
                 .await?;
         }
 
@@ -222,10 +222,10 @@ pub async fn dc_get_oauth2_access_token(
 
 pub async fn dc_get_oauth2_addr(
     context: &Context,
-    addr: impl AsRef<str>,
-    code: impl AsRef<str>,
+    addr: &str,
+    code: &str,
 ) -> Result<Option<String>> {
-    let oauth2 = match Oauth2::from_address(addr.as_ref()).await {
+    let oauth2 = match Oauth2::from_address(addr).await {
         Some(o) => o,
         None => return Ok(None),
     };
@@ -233,16 +233,14 @@ pub async fn dc_get_oauth2_addr(
         return Ok(None);
     }
 
-    if let Some(access_token) =
-        dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false).await?
-    {
-        let addr_out = oauth2.get_addr(context, access_token).await;
+    if let Some(access_token) = dc_get_oauth2_access_token(context, addr, code, false).await? {
+        let addr_out = oauth2.get_addr(context, &access_token).await;
         if addr_out.is_none() {
             // regenerate
             if let Some(access_token) =
                 dc_get_oauth2_access_token(context, addr, code, true).await?
             {
-                Ok(oauth2.get_addr(context, access_token).await)
+                Ok(oauth2.get_addr(context, &access_token).await)
             } else {
                 Ok(None)
             }
@@ -255,8 +253,8 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
-        let addr_normalized = normalize_addr(addr.as_ref());
+    async fn from_address(addr: &str) -> Option<Self> {
+        let addr_normalized = normalize_addr(addr);
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
@@ -274,9 +272,9 @@ impl Oauth2 {
         None
     }
 
-    async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
+    async fn get_addr(&self, context: &Context, access_token: &str) -> Option<String> {
         let userinfo_url = self.get_userinfo.unwrap_or("");
-        let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
+        let userinfo_url = replace_in_uri(userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as
         // {
@@ -309,7 +307,7 @@ impl Oauth2 {
     }
 }
 
-async fn is_expired(context: &Context) -> Result<bool, crate::sql::Error> {
+async fn is_expired(context: &Context) -> Result<bool> {
     let expire_timestamp = context
         .sql
         .get_raw_config_int64("oauth2_timestamp_expires")
@@ -326,9 +324,9 @@ async fn is_expired(context: &Context) -> Result<bool, crate::sql::Error> {
     Ok(true)
 }
 
-fn replace_in_uri(uri: impl AsRef<str>, key: impl AsRef<str>, value: impl AsRef<str>) -> String {
-    let value_urlencoded = utf8_percent_encode(value.as_ref(), NON_ALPHANUMERIC).to_string();
-    uri.as_ref().replace(key.as_ref(), &value_urlencoded)
+fn replace_in_uri(uri: &str, key: &str, value: &str) -> String {
+    let value_urlencoded = utf8_percent_encode(value, NON_ALPHANUMERIC).to_string();
+    uri.replace(key, &value_urlencoded)
 }
 
 fn normalize_addr(addr: &str) -> &str {

src/param.rs

@@ -306,8 +306,8 @@ impl Params {
         let file = ParamsFile::from_param(context, val)?;
         let blob = match file {
             ParamsFile::FsPath(path) => match create {
-                true => BlobObject::new_from_path(context, path).await?,
-                false => BlobObject::from_path(context, path)?,
+                true => BlobObject::new_from_path(context, &path).await?,
+                false => BlobObject::from_path(context, &path)?,
             },
             ParamsFile::Blob(blob) => blob,
         };