also export qoutes, message state and is_forwarded

step, this could decrease memory usage

.circleci/config.yml

@@ -7,116 +7,13 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
             
-
-restore-workspace: &restore-workspace
-  attach_workspace:
-    at: /mnt
-
-restore-cache: &restore-cache
-  restore_cache:
-    keys:
-      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-      - repo-source-{{ .Branch }}-{{ .Revision }}
-
-commands:
-  test_target:
-    parameters:
-      target:
-        type: string
-    steps:
-      - *restore-workspace
-      - *restore-cache
-      - run:
-          name: Test (<< parameters.target >>)
-          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
-          no_output_timeout: 15m
-
 jobs:
-  cargo_fetch:
-    executor: default
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-      - run: rustup install $(cat rust-toolchain)
-      - run: rustup default $(cat rust-toolchain)
-      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
-      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
-      - run: cargo fetch
-      - run: rustc +stable --version
-      - run: rustc +$(cat rust-toolchain) --version
-      # make sure this git repo doesn't grow too big
-      - run: git gc
-      - persist_to_workspace:
-          root: /mnt
-          paths:
-            - crate
-      - save_cache:
-          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
-          paths:
-            - "~/.cargo"
-            - "~/.rustup"
-
-  rustfmt:
-    executor: default
-    steps:
-      - *restore-workspace
-      - *restore-cache
-      - run:
-          name: Run cargo fmt
-          command: cargo fmt --all -- --check
-
-  test_macos:
-    macos:
-      xcode: "10.0.0"
-    working_directory: ~/crate
-    steps:
-      - run:
-          name: Configure environment variables
-          command: |
-            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
-            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
-      - checkout
-      - run:
-          name: Install Rust
-          command: |
-            curl https://sh.rustup.rs -sSf | sh -s -- -y
-      - run: rustup install $(cat rust-toolchain)
-      - run: rustup default $(cat rust-toolchain)
-      - run: cargo fetch
-      - run:
-          name: Test
-          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
-
-  test_x86_64-unknown-linux-gnu:
-    executor: default
-    steps:
-      - test_target:
-          target: "x86_64-unknown-linux-gnu"
-
-  test_i686-unknown-linux-gnu:
-    executor: default
-    steps:
-      - test_target:
-          target: "i686-unknown-linux-gnu"
-
-  test_aarch64-linux-android:
-    executor: default
-    steps:
-      - test_target:
-          target: "aarch64-linux-android"
-
-
   build_doxygen:
     executor: doxygen
     steps:
       - checkout
-      - run: bash ci_scripts/run-doxygen.sh
+      - run: bash scripts/run-doxygen.sh
       - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
@@ -130,7 +27,10 @@ jobs:
       - checkout
       # the following commands on success produces
       # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
+      - run:
+          # building aarch64 packages under qemu is very slow
+          no_output_timeout: 60m
+          command: bash scripts/remote_python_packaging.sh
       - persist_to_workspace:
           root: workspace
           paths:
@@ -138,102 +38,40 @@ jobs:
             - py-docs
             - wheelhouse
 
-  remote_tests_rust:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_rust.sh
-
-  remote_tests_python:
-    machine: true
-    steps:
-      - checkout
-      - run: ci_scripts/remote_tests_python.sh
-
   upload_docs_wheels:
     machine: true
     steps:
       - checkout
       - attach_workspace:
           at: workspace
-      - run: pyenv versions
-      - run: pyenv global 3.5.2
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
-
-  clippy:
-    executor: default
-    steps:
-      - *restore-workspace
-      - *restore-cache
-      - run:
-          name: Run cargo clippy
-          command: cargo clippy 
-
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
 workflows:
   version: 2.1
 
   test:
     jobs:
-      # - cargo_fetch
-
-      - remote_tests_rust:
-          filters:
-            tags:
-              only: /.*/
-
-      - remote_tests_python:
-          filters:
-            tags:
-              only: /.*/
-
       - remote_python_packaging:
           filters:
-            branches:
-              only: master
             tags:
-              only: /.*/
+              only: /.+/
+            branches:
+              ignore: /.*/
+
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
 
       - upload_docs_wheels:
           requires:
              - remote_python_packaging
              - build_doxygen
           filters:
-            branches:
-              only: master
             tags:
-              only: /.*/
-      # - rustfmt:
-      #     requires:
-      #       - cargo_fetch
-      # - clippy:
-      #     requires:
-      #      - cargo_fetch
-
-      - build_doxygen:
-          filters:
+              only: /.+/
             branches:
-              only: master
-            tags:
-              only: /.*/
-
-      # Linux Desktop 64bit
-      # - test_x86_64-unknown-linux-gnu:
-      #     requires:
-      #       - cargo_fetch
-
-      # Linux Desktop 32bit
-      # - test_i686-unknown-linux-gnu:
-      #     requires:
-      #       - cargo_fetch
-
-      # Android 64bit
-      # - test_aarch64-linux-android:
-      #     requires:
-      #       - cargo_fetch
-
-      # Desktop Apple
-      # - test_macos:
-      #     requires:
-      #       - cargo_fetch
+              ignore: /.*/

.gitattributes

@@ -12,3 +12,6 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
+*.py diff=python
+*.rs diff=rust
+*.md diff=markdown

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    commit-message:
+      prefix: "cargo"
+    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -14,11 +14,11 @@ jobs:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.45.0
+          toolchain: 1.50.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -29,10 +29,10 @@ jobs:
   run_clippy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: 1.45.0
+            toolchain: 1.50.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -40,29 +40,41 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --workspace --tests --examples
 
+  docs:
+    name: Rust doc comments
+    runs-on: ubuntu-latest
+    env:
+      RUSTDOCFLAGS: -Dwarnings
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - name: Install rust stable toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          components: rust-docs
+          override: true
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v1
+      - name: Rustdoc
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        # macOS disabled due to random failures related to caching
-        #os: [ubuntu-latest, windows-latest, macOS-latest]
-        os: [ubuntu-latest, windows-latest]
-        rust: [1.45.0]
-        experimental: [false]
-#        include:
-#          - os: ubuntu-latest
-#            rust: nightly
-#            experimental: true
-#          - os: windows-latest
-#            rust: nightly
-#            experimental: true
-#          - os: macOS-latest
-#            rust: nightly
-#            experimental: true
-
+        include:
+          - os: ubuntu-latest
+            rust: 1.50.0
+            python: 3.6
+          - os: windows-latest
+            rust: 1.50.0
+            python: false # Python bindings compilation on Windows is not supported.
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -92,8 +104,10 @@ jobs:
 
     - name: check
       uses: actions-rs/cargo@v1
+      env:
+        RUSTFLAGS: -D warnings
       with:
-        command:  check
+        command: check
         args: --all --bins --examples --tests --features repl
 
     - name: tests
@@ -101,3 +115,29 @@ jobs:
       with:
         command: test
         args: --all
+
+    - name: install python
+      if: ${{ matrix.python }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: install tox
+      if: ${{ matrix.python }}
+      run: pip install tox
+
+    - name: build C library
+      if: ${{ matrix.python }}
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: -p deltachat_ffi
+
+    - name: run python tests
+      if: ${{ matrix.python }}
+      env:
+        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        DCC_RS_TARGET: debug
+        DCC_RS_DEV: ${{ github.workspace }}
+      working-directory: python
+      run: tox -e lint,doc,py3

.github/workflows/repl.yml

@@ -0,0 +1,32 @@
+# Manually triggered action to build a Windows repl.exe which users can
+# download to debug complex bugs.
+
+name: Build Windows REPL .exe
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_repl:
+    name: Build REPL example
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: 1.50.0
+        override: true
+
+    - name: build
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --example repl --features repl,vendored
+
+    - name: Upload binary
+      uses: actions/upload-artifact@v2
+      with:
+        name: repl.exe
+        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,5 +1,6 @@
 /target
 **/*.rs.bk
+/build
 
 # ignore vi temporaries
 *~
@@ -25,3 +26,5 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
+
+coverage/

CHANGELOG.md

@@ -1,5 +1,276 @@
 # Changelog 
 
+## 1.55.0
+
+- fix panic when receiving some HTML messages #2434
+
+- fix downloading some messages multiple times #2430
+
+- fix formatting of read receipt texts #2431
+
+- simplify SQL error handling #2415
+
+- explicit rust API for creating chats with blocked status #2282
+
+- debloat the binary by using less AsRef arguments #2425
+
+
+## 1.54.0
+
+- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
+
+- global search performance improvement #2364 #2365 #2366
+
+- improve SQLite performance with `PRAGMA synchronous=normal` #2382
+
+- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
+
+- python: list `requests` as a requirement #2390
+
+- fix creation of many delete jobs when being offline #2372
+
+- synchronize status between devices #2386
+
+- deaddrop (contact requests) chat improvements #2373
+
+- add "Forwarded:" to notification and chatlist summaries #2310
+
+- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
+
+- improve tests #2360 #2362 #2370 #2377 #2387
+
+- cleanup #2359 #2361 #2374 #2376 #2379 #2388
+
+
+## 1.53.0
+
+- fix sqlx performance regression #2355 2356
+
+- add a `ci_scripts/coverage.sh` #2333 #2334
+
+- refactorings and tests #2348 #2349 #2350
+
+- improve python bindings #2332 #2326
+
+
+## 1.52.0
+
+- database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340
+
+- add alias support: UIs should check for `dc_msg_get_override_sender_name()`
+  also in single-chats now and display divergent names and avatars #2297
+
+- parse blockquote-tags for better quote detection #2313
+
+- ignore unknown classical emails from spam folder #2311
+
+- support "Mixed Up” encryption repairing #2321
+
+- fix single chat search #2344
+
+- fix nightly clippy and rustc errors #2341
+
+- update dependencies #2350
+
+- improve ci #2342
+
+- improve python bindings #2332 #2326
+
+
+## 1.51.0
+
+- breaking change: You have to call `dc_stop_io()`/`dc_start_io()`
+  before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`:
+  fix race condition and db corruption
+  when a message was received during backup #2253
+
+- save subject for messages: new api `dc_msg_get_subject()`,
+  when quoting, use the subject of the quoted message as the new subject,
+  instead of the last subject in the chat #2274 #2283
+
+- new apis to get full or html message,
+  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151 #2264 #2279
+
+- new chat type and apis for the new mailing list support,
+  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
+  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
+  #2241 #2243 #2258 #2259 #2261 #2267 #2270 #2272 #2290
+
+- new api `dc_decide_on_contact_request()`,
+  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
+
+- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
+
+- new api `dc_get_chat_encrinfo()` #2186
+
+- new api `dc_contact_get_status()`, returning the recent footer #2218 #2307
+
+- improve contact name update rules,
+  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
+
+- new api for bots: `dc_msg_set_html()` #2153
+
+- new api for bots: `dc_msg_set_override_sender_name()` #2231
+
+- api removed: `dc_is_io_running()` #2139
+
+- api removed: `dc_contact_get_first_name()` #2165 #2171
+
+- improve compatibility with providers changing the Message-ID
+  (as Outlook.com) #2250 #2265
+
+- correctly show emails that were sent to an alias and then bounced 
+
+- implement Consistent Color Generation (XEP-0392),
+  that results in contact colors be be changed #2228 #2229 #2239
+
+- fetch recent existing messages
+  and create corresponding chats after configure #2106
+
+- improve e-mail compatibility
+  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
+
+- better support videochat-services not supporting random rooms #2191
+
+- export backups as .tar files #2023
+
+- scale avatars based on media_quality, fix avatar rotation #2063
+
+- compare ephemeral timer to parent message to deal with reordering better #2100
+
+- better ephemeral system messages #2183
+
+- read quotes out of html messages #2104
+
+- prepend subject to messages with attachments, if needed #2111
+
+- run housekeeping at least once a day #2114
+
+- resolve MX domain only once per OAuth2 provider #2122
+
+- configure provider based on MX record #2123 #2134
+
+- make transient bad destination address error permanent
+  after n tries #2126 #2202
+
+- enable strict TLS for known providers by default #2121
+
+- improve and harden secure join #2154 #2161 #2251
+
+- update `dc_get_info()` to return more information #2156
+
+- prefer In-Reply-To/References
+  over group-id stored in Message-ID #2164 #2172 #2173
+
+- apply gossiped encryption preference to new peerstates #2174
+
+- fix: do not return quoted messages from the trash chat #2221
+
+- fix: allow emojis for location markers #2177
+
+- fix encoding of Chat-Group-Name-Changed messages that could even lead to
+  messages not being delivered #2141
+
+- fix error when no temporary directory is available #1929
+
+- fix marking read receipts as seen #2117
+
+- fix read-notification for mixed-case addresses #2103
+
+- fix decoding of attachment filenames #2080 #2094 #2102
+
+- fix downloading ranges of message #2061
+
+- fix parsing quoted encoded words in From: header #2193 #2204
+
+- fix import/export race condition #2250
+
+- fix: exclude muted chats from notified-list #2269 #2275
+
+- fix: update uid_next if the server rewind it #2288
+
+- fix: return error on fingerprint mismatch on qr-scan #2295
+
+- fix ci #2217 #2226 #2244 #2245 #2249 #2277 #2286
+
+- try harder on backup opening #2148
+
+- trash messages more thoroughly #2273
+
+- nicer logging #2284
+
+- add CMakeLists.txt #2260
+
+- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
+
+- improve python bindings #2113 #2115 #2133 #2214
+
+- improve documentation #2143 #2160 #2175 #2146
+
+- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
+  #2200 #2230 #2262 #2203
+
+- update provider-database #2299
+
+
+## 1.50.0
+
+- do not fetch emails in between inbox_watch disabled and enabled again #2087
+
+- fix: do not fetch from INBOX if inbox_watch is disabled #2085
+
+- fix: do not use STARTTLS when PLAIN connection is requested
+  and do not allow downgrade if STARTTLS is not available #2071
+
+
+## 1.49.0
+
+- add timestamps to image and video filenames #2068
+
+- forbid quoting messages from another context #2069
+
+- fix: preserve quotes in messages with attachments #2070
+
+
+## 1.48.0
+
+- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
+  #2035 #2042
+
+- skip fetch existing messages/contacts if config-option `bot` set #2017
+
+- always log why a message is sorted to trash #2045
+
+- display a quote if top posting is detected #2047
+
+- add ephemeral task cancellation to `dc_stop_io()`;
+  before, there was no way to quickly terminate pending ephemeral tasks #2051
+
+- when saved-messages chat is deleted,
+  a device-message about recreation is added #2050
+
+- use `max_smtp_rcpt_to` from provider-db,
+  sending messages to many recipients in configurable chunks #2056
+
+- fix handling of empty autoconfigure files #2027
+
+- fix adding saved messages to wrong chats on multi-device #2034 #2039
+
+- fix hang on android4.4 and other systems
+  by adding a workaround to executer-blocking-handling bug #2040
+
+- fix secret key export/import roundtrip #2048
+
+- fix mistakenly unarchived chats #2057
+
+- fix outdated-reminder test that fails only 7 days a year,
+  including halloween :) #2059
+
+- improve python bindings #2021 #2036 #2038
+
+- update provider-database #2037
+
+
 ## 1.47.0
 
 - breaking change: `dc_update_device_chats()` removed;
@@ -27,6 +298,7 @@
 
 - configure now collects recent contacts and fetches last messages
   unless disabled by `fetch_existing` config-option #1913 #2003
+  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
 
 - emit `DC_EVENT_CHAT_MODIFIED` on contact rename
   and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
@@ -40,6 +312,8 @@
 
 - mark all failed messages as failed when receiving an NDN #1993
 
+- check some easy cases for bad system clock and outdated app #1901
+
 - fix import temporary directory usage #1929
 
 - fix forcing encryption for reset peers #1998

CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.16)
+project(deltachat LANGUAGES C)
+
+find_program(CARGO cargo)
+
+add_custom_command(
+	OUTPUT
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --release --no-default-features
+
+	# Build in `deltachat-ffi` directory instead of using
+	# `--package deltachat_ffi` to avoid feature resolver version
+	# "1" bug which makes `--no-default-features` affect only
+	# `deltachat`, but not `deltachat-ffi` package.
+	#
+	# We can't enable version "2" resolver [1] because it is not
+	# stable yet on rust 1.50.0.
+	#
+	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
+)
+
+add_custom_target(
+	lib_deltachat
+	ALL
+	DEPENDS
+	"target/release/libdeltachat.a"
+	"target/release/libdeltachat.so"
+	"target/release/pkgconfig/deltachat.pc"
+)
+
+include(GNUInstallDirs)
+install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,83 +1,90 @@
 [package]
 name = "deltachat"
-version = "1.47.0"
+version = "1.55.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
+[profile.dev]
+debug = 0
+
 [profile.release]
 lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-libc = "0.2.51"
-pgp = { version = "0.7.0", default-features = false } 
+ansi_term = { version = "0.12.1", optional = true }
+anyhow = "1.0.40"
+async-imap = "0.5.0"
+async-native-tls = { version = "0.3.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
+async-std-resolver = "0.20.3"
+async-std = { version = "~1.9.0", features = ["unstable"] }
+async-tar = "0.3.0"
+async-trait = "0.1.50"
+backtrace = "0.3.59"
+base64 = "0.13"
+bitflags = "1.1.0"
+byteorder = "1.3.1"
+charset = "0.1"
+chrono = "0.4.6"
+dirs = { version = "3.0.2", optional=true }
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+escaper = "0.1.1"
+futures = "0.3.15"
 hex = "0.4.0"
-sha2 = "0.9.0"
-rand = "0.7.0"
-smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+indexmap = "1.3.0"
+itertools = "0.10.0"
+kamadak-exif = "0.5"
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+libc = "0.2.95"
+log = {version = "0.4.8", optional = true }
+mailparse = "0.13.4"
+native-tls = "0.2.3"
+num_cpus = "1.13.0"
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.4.0"
-async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.4", features = ["unstable"] }
-base64 = "0.12"
-charset = "0.1"
-percent-encoding = "2.0"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-chrono = "0.4.6"
-indexmap = "1.3.0"
-kamadak-exif = "0.5"
 once_cell = "1.4.1"
-regex = "1.1.6"
-rusqlite = { version = "0.24", features = ["bundled"] }
-r2d2_sqlite = "0.17.0"
-r2d2 = "0.8.5"
-strum = "0.19.0"
-strum_macros = "0.19.0"
-backtrace = "0.3.33"
-byteorder = "1.3.1"
-itertools = "0.9.0"
-quick-xml = "0.18.1"
-escaper = "0.1.0"
-bitflags = "1.1.0"
-sanitize-filename = "0.3.0"
-stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.13.0"
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-native-tls = "0.2.3"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-futures = "0.3.4"
-thiserror = "1.0.14"
-anyhow = "1.0.28"
-async-trait = "0.1.31"
-url = "2.1.1"
-async-std-resolver = "0.19.5"
-async-tar = "0.3.0"
-uuid = { version = "0.8", features = ["serde", "v4"] }
-
+percent-encoding = "2.0"
+pgp = { version = "0.7.0", default-features = false }
 pretty_env_logger = { version = "0.4.0", optional = true }
-log = {version = "0.4.8", optional = true }
-rustyline = { version = "4.1.0", optional = true }
-ansi_term = { version = "0.12.1", optional = true }
-dirs = { version = "3.0.1", optional=true }
+quick-xml = "0.22.0"
+r2d2 = "0.8.9"
+r2d2_sqlite = "0.18.0"
+rand = "0.7.0"
+regex = "1.4.6"
+rusqlite = "0.25"
+rust-hsluv = "0.1.4"
+rustyline = { version = "8.2.0", optional = true }
+sanitize-filename = "0.3.0"
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+sha-1 = "0.9.6"
+sha2 = "0.9.5"
+smallvec = "1.0.0"
+stop-token = "0.2.0"
+strum = "0.20.0"
+strum_macros = "0.20.1"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+thiserror = "1.0.25"
 toml = "0.5.6"
-
+url = "2.2.2"
+uuid = { version = "0.8", features = ["serde", "v4"] }
+zip = "0.5.12"
 
 [dev-dependencies]
-tempfile = "3.0"
-pretty_assertions = "0.6.1"
-pretty_env_logger = "0.4.0"
-proptest = "0.10"
-async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
-futures-lite = "1.7.0"
+ansi_term = "0.12.0"
+async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
 criterion = "0.3"
+futures-lite = "1.7.0"
+log = "0.4.11"
+pretty_assertions = "0.7.2"
+pretty_env_logger = "0.4.0"
+proptest = "1.0"
+tempfile = "3.0"
 
 [workspace]
 members = [
@@ -100,10 +107,17 @@ required-features = ["repl"]
 name = "create_account"
 harness = false
 
+[[bench]]
+name = "contacts"
+harness = false
+
+[[bench]]
+name = "search_msgs"
+harness = false
+
 [features]
 default = []
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
 nightly = ["pgp/nightly"]
-

README.md

@@ -2,7 +2,8 @@
 
 > Deltachat-core written in Rust 
 
-[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
+[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -17,7 +18,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -95,7 +96,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests
@@ -111,11 +112,6 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
-[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
-[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
-[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
-[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
 ## Language bindings and frontend projects
 
 Language bindings are available for:
@@ -123,7 +119,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
+- [Go](https://github.com/deltachat/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

benches/contacts.rs

@@ -0,0 +1,39 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use tempfile::tempdir;
+
+async fn address_book_benchmark(n: u32, read_count: u32) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    let book = (0..n)
+        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .collect::<Vec<String>>()
+        .join("");
+
+    Contact::add_address_book(&context, &book).await.unwrap();
+
+    let query: Option<&str> = None;
+    for _ in 0..read_count {
+        Contact::get_all(&context, 0, query).await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 500 contacts", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+    });
+
+    c.bench_function("create 100 contacts and read it 1000 times", |b| {
+        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -0,0 +1,29 @@
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::context::Context;
+use std::path::Path;
+
+async fn search_benchmark(path: impl AsRef<Path>) {
+    let dbfile = path.as_ref();
+    let id = 100;
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
+
+    for _ in 0..10u32 {
+        context.search_msgs(None, "hello").await.unwrap();
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        c.bench_function("search hello", |b| {
+            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+        });
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/ci_upload.sh

@@ -1,57 +0,0 @@
-#!/bin/bash
-
-if [ -z "$DEVPI_LOGIN" ] ; then 
-    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
-    exit 0
-fi
-
-set -xe
-
-PYDOCDIR=${1:?directory with python docs}
-WHEELHOUSEDIR=${2:?directory with pre-built wheels}
-DOXYDOCDIR=${3:?directory where doxygen docs to be found}
-
-export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
-
-
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
-
-# C docs to c.delta.chat
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$DOXYDOCDIR/html/" \
-  delta@c.delta.chat:build-c/${BRANCH}
-
-echo -----------------------
-echo upload wheels 
-echo -----------------------
-
-# Bundle external shared libraries into the wheels
-pushd $WHEELHOUSEDIR
-
-pip3 install -U pip setuptools
-pip3 install devpi-client
-devpi use https://m.devpi.net
-devpi login dc --password $DEVPI_LOGIN
-
-N_BRANCH=${BRANCH//[\/]}
-
-devpi use dc/$N_BRANCH || {
-    devpi index -c $N_BRANCH 
-    devpi use dc/$N_BRANCH
-}
-devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*
-
-popd
-
-# remove devpi non-master dc indices if thy are too old
-python ci_scripts/cleanup_devpi_indices.py

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.5 environment as the base environment 
-/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.5.*/bin/tox
-ln -s /opt/_internal/cpython-3.5.*/bin/devpi
-ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
-
-popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
-export PATH=/root/.cargo/bin:$PATH
-rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -xe
-export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
-export CIRCLE_BUILD_NUM=$USER
-export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
-export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
-
-time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -1,77 +0,0 @@
-name: CI
-
-on:
-  pull_request:
-  push:
-
-env:
-  RUSTFLAGS: -Dwarnings
-
-jobs:
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly]
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      if: matrix.rust == 'nightly'
-      with:
-        command:  check
-        args: --all --bins --examples --tests
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
-
-    - name: tests ignored
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all --release -- --ignored
-
-  check_fmt:
-    name: Checking fmt and docs
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - uses: actions-rs/toolchain@v1
-      with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-
-    - name: fmt
-      run: cargo fmt --all -- --check
-
-#  clippy_check:
-#    name: Clippy check
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - uses: actions/checkout@v1
-#    - uses: actions-rs/toolchain@v1
-#      with:
-#          profile: minimal
-#          toolchain: nightly
-#          override: true
-#          components: clippy
-#
-#    - name: clippy
-#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library typically run in a docker
-# container that contains all library deps but should also work
-# outside if you have the dependencies installed on your system.
-
-set -e -x
-
-# Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
-
-# install core lib
-
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
-
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
-
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-if [ -n "$TESTS" ]; then
-
-    pushd python
-    # prepare a clean tox run
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-
-    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
-    # allows running of "liveconfig" tests but for speed reasons
-    # we run them only for the highest python version we support
-    
-    # we split out qr-tests run to minimize likelyness of flaky tests
-    # (some qr tests are pretty heavy in terms of send/received
-    # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
-    unset DCC_PY_LIVECONFIG
-    unset DCC_NEW_TMP_EMAIL
-    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
-    tox --workdir "$TOXWORKDIR" -e auditwheels
-    popd
-fi
-
-
-# if [ -n "$DOCS" ]; then
-#     echo -----------------------
-#     echo generating python docs
-#     echo -----------------------
-#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-# fi

ci_scripts/remote_tests_rust.sh

@@ -1,30 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-set -e
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-    export RUSTC_WRAPPER=\`which sccache\`
-    cd $BUILDDIR
-    export TARGET=x86_64-unknown-linux-gnu
-    export RUSTC_WRAPPER=sccache
-
-    bash ci_scripts/run-rust-test.sh
-_HERE
-

ci_scripts/run-doxygen.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-cd deltachat-ffi 
-PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.47.0"
+version = "1.55.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,9 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.6.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
+async-std = "1.9.0"
+anyhow = "1.0.40"
+thiserror = "1.0.25"
 rand = "0.7.3"
 
 [features]

deltachat-ffi/DoxygenLayout.xml

@@ -0,0 +1,38 @@
+<doxygenlayout version="1.0">
+  <!-- Generated by doxygen 1.8.20 -->
+  <!-- Navigation index tabs for HTML output -->
+  <navindex>
+    <tab type="mainpage" visible="yes" title=""/>
+    <tab type="classes" visible="yes" title="">
+      <tab type="classlist" visible="no" title="" intro=""/>
+      <tab type="classindex" visible="no" title=""/>
+      <tab type="hierarchy" visible="no" title="" intro=""/>
+      <tab type="classmembers" visible="no" title="" intro=""/>
+    </tab>
+    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="pages" visible="yes" title="" intro=""/>
+    <tab type="namespaces" visible="yes" title="">
+      <tab type="namespacelist" visible="yes" title="" intro=""/>
+      <tab type="namespacemembers" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="interfaces" visible="yes" title="">
+      <tab type="interfacelist" visible="yes" title="" intro=""/>
+      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="structs" visible="yes" title="">
+      <tab type="structlist" visible="yes" title="" intro=""/>
+      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
+    </tab>
+    <tab type="exceptions" visible="yes" title="">
+      <tab type="exceptionlist" visible="yes" title="" intro=""/>
+      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
+      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="files" visible="yes" title="">
+      <tab type="filelist" visible="yes" title="" intro=""/>
+      <tab type="globals" visible="yes" title="" intro=""/>
+    </tab>
+    <tab type="examples" visible="yes" title="" intro=""/>
+  </navindex>
+</doxygenlayout>

deltachat-ffi/src/lib.rs

@@ -21,10 +21,10 @@ use std::ptr;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
+use anyhow::Context as _;
 use async_std::task::{block_on, spawn};
 use num_traits::{FromPrimitive, ToPrimitive};
 
-use deltachat::accounts::Accounts;
 use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, Origin};
@@ -32,8 +32,9 @@ use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
-use deltachat::stock::StockMessage;
+use deltachat::stock_str::StockMessage;
 use deltachat::*;
+use deltachat::{accounts::Accounts, log::LogExt};
 
 mod dc_array;
 
@@ -130,12 +131,14 @@ pub unsafe extern "C" fn dc_set_config(
         return 0;
     }
     let ctx = &*context;
-    match config::Config::from_str(&to_string_lossy(key)) {
-        // When ctx.set_config() fails it already logged the error.
-        // TODO: Context::set_config() should not log this
+    let key = to_string_lossy(key);
+    match config::Config::from_str(&key) {
         Ok(key) => block_on(async move {
-            ctx.set_config(key, to_opt_string_lossy(value).as_deref())
+            let value = to_opt_string_lossy(value);
+            ctx.set_config(key, value.as_deref())
                 .await
+                .with_context(|| format!("Can't set {} to {:?}", key, value))
+                .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
         }),
         Err(_) => {
@@ -156,7 +159,14 @@ pub unsafe extern "C" fn dc_get_config(
     }
     let ctx = &*context;
     match config::Config::from_str(&to_string_lossy(key)) {
-        Ok(key) => block_on(async move { ctx.get_config(key).await.unwrap_or_default().strdup() }),
+        Ok(key) => block_on(async move {
+            ctx.get_config(key)
+                .await
+                .log_err(ctx, "Can't get config")
+                .unwrap_or_default()
+                .unwrap_or_default()
+                .strdup()
+        }),
         Err(_) => {
             warn!(ctx, "dc_get_config(): invalid key");
             "".strdup()
@@ -225,8 +235,13 @@ pub unsafe extern "C" fn dc_get_info(context: *const dc_context_t) -> *mut libc:
     }
     let ctx = &*context;
     block_on(async move {
-        let info = ctx.get_info().await;
-        render_info(info).unwrap_or_default().strdup()
+        match ctx.get_info().await {
+            Ok(info) => render_info(info).unwrap_or_default().strdup(),
+            Err(err) => {
+                warn!(ctx, "failed to get info: {}", err);
+                "".strdup()
+            }
+        }
     })
 }
 
@@ -256,7 +271,7 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     let redirect = to_string_lossy(redirect);
 
     block_on(async move {
-        match oauth2::dc_get_oauth2_url(&ctx, addr, redirect).await {
+        match oauth2::dc_get_oauth2_url(&ctx, &addr, &redirect).await {
             Some(res) => res.strdup(),
             None => ptr::null_mut(),
         }
@@ -283,7 +298,12 @@ pub unsafe extern "C" fn dc_is_configured(context: *mut dc_context_t) -> libc::c
     }
     let ctx = &*context;
 
-    block_on(async move { ctx.is_configured().await as libc::c_int })
+    block_on(async move {
+        ctx.is_configured()
+            .await
+            .log_err(ctx, "failed to get configured state")
+            .unwrap_or_default() as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -296,16 +316,6 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     block_on(ctx.start_io())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_is_io_running(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(ctx.is_io_running()) as libc::c_int
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_id(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -629,7 +639,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        chat::create_by_contact_id(&ctx, contact_id)
+        ChatId::create_for_contact(&ctx, contact_id)
             .await
             .log_err(ctx, "Failed to create chat from contact_id")
             .map(|id| id.to_u32())
@@ -649,11 +659,12 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     let ctx = &*context;
 
     block_on(async move {
-        chat::get_by_contact_id(&ctx, contact_id)
+        ChatId::lookup_by_contact(&ctx, contact_id)
             .await
             .log_err(ctx, "Failed to get chat for contact_id")
+            .unwrap_or_default() // unwraps the Result
             .map(|id| id.to_u32())
-            .unwrap_or(0)
+            .unwrap_or(0) // unwraps the Option
     })
 }
 
@@ -778,7 +789,12 @@ pub unsafe extern "C" fn dc_set_draft(
         Some(&mut ffi_msg.message)
     };
 
-    block_on(ChatId::new(chat_id).set_draft(&ctx, msg))
+    block_on(async move {
+        ChatId::new(chat_id)
+            .set_draft(&ctx, msg)
+            .await
+            .unwrap_or_log_default(ctx, "failed to set draft");
+    });
 }
 
 #[no_mangle]
@@ -873,6 +889,7 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
         Box::into_raw(Box::new(
             chat::get_chat_msgs(&ctx, ChatId::new(chat_id), flags, marker_flag)
                 .await
+                .unwrap_or_log_default(ctx, "failed to get chat msgs")
                 .into(),
         ))
     })
@@ -886,7 +903,12 @@ pub unsafe extern "C" fn dc_get_msg_cnt(context: *mut dc_context_t, chat_id: u32
     }
     let ctx = &*context;
 
-    block_on(async move { ChatId::new(chat_id).get_msg_cnt(&ctx).await as libc::c_int })
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_msg_cnt(&ctx)
+            .await
+            .unwrap_or_log_default(ctx, "failed to get msg count") as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -900,7 +922,12 @@ pub unsafe extern "C" fn dc_get_fresh_msg_cnt(
     }
     let ctx = &*context;
 
-    block_on(async move { ChatId::new(chat_id).get_fresh_msg_cnt(&ctx).await as libc::c_int })
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_fresh_msg_cnt(&ctx)
+            .await
+            .unwrap_or_log_default(ctx, "failed to get fresh msg cnt") as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -935,6 +962,8 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
         let arr = dc_array_t::from(
             ctx.get_fresh_msgs()
                 .await
+                .log_err(ctx, "Failed to get fresh messages")
+                .unwrap_or_default()
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
                 .collect::<Vec<u32>>(),
@@ -996,6 +1025,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
                 or_msg_type3,
             )
             .await
+            .unwrap_or_log_default(ctx, "Failed get_chat_media")
             .into(),
         ))
     })
@@ -1037,7 +1067,7 @@ pub unsafe extern "C" fn dc_get_next_media(
             or_msg_type3,
         )
         .await
-        .map(|msg_id| msg_id.to_u32())
+        .map(|msg_id| msg_id.map(|id| id.to_u32()).unwrap_or_default())
         .unwrap_or(0)
     })
 }
@@ -1130,7 +1160,11 @@ pub unsafe extern "C" fn dc_get_chat_contacts(
     let ctx = &*context;
 
     block_on(async move {
-        let arr = dc_array_t::from(chat::get_chat_contacts(&ctx, ChatId::new(chat_id)).await);
+        let arr = dc_array_t::from(
+            chat::get_chat_contacts(&ctx, ChatId::new(chat_id))
+                .await
+                .unwrap_or_log_default(ctx, "Failed get_chat_contacts"),
+        );
         Box::into_raw(Box::new(arr))
     })
 }
@@ -1146,11 +1180,17 @@ pub unsafe extern "C" fn dc_search_msgs(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
         let arr = dc_array_t::from(
-            ctx.search_msgs(ChatId::new(chat_id), to_string_lossy(query))
+            ctx.search_msgs(chat_id, &to_string_lossy(query))
                 .await
+                .unwrap_or_log_default(ctx, "Failed search_msgs")
                 .iter()
                 .map(|msg_id| msg_id.to_u32())
                 .collect::<Vec<u32>>(),
@@ -1197,7 +1237,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     };
 
     block_on(async move {
-        chat::create_group_chat(&ctx, protect, to_string_lossy(name))
+        chat::create_group_chat(&ctx, protect, &to_string_lossy(name))
             .await
             .log_err(ctx, "Failed to create group chat")
             .map(|id| id.to_u32())
@@ -1264,14 +1304,15 @@ pub unsafe extern "C" fn dc_set_chat_name(
     chat_id: u32,
     name: *const libc::c_char,
 ) -> libc::c_int {
-    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL as u32 || name.is_null() {
+    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL.to_u32() || name.is_null()
+    {
         eprintln!("ignoring careless call to dc_set_chat_name()");
         return 0;
     }
     let ctx = &*context;
 
     block_on(async move {
-        chat::set_chat_name(&ctx, ChatId::new(chat_id), to_string_lossy(name))
+        chat::set_chat_name(&ctx, ChatId::new(chat_id), &to_string_lossy(name))
             .await
             .map(|_| 1)
             .unwrap_or_log_default(&ctx, "Failed to set chat name")
@@ -1284,7 +1325,7 @@ pub unsafe extern "C" fn dc_set_chat_profile_image(
     chat_id: u32,
     image: *const libc::c_char,
 ) -> libc::c_int {
-    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL as u32 {
+    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL.to_u32() {
         eprintln!("ignoring careless call to dc_set_chat_profile_image()");
         return 0;
     }
@@ -1332,6 +1373,29 @@ pub unsafe extern "C" fn dc_set_chat_mute_duration(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_chat_encrinfo(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_chat_encrinfo()");
+        return "".strdup();
+    }
+    let ctx = &*context;
+
+    block_on(async move {
+        ChatId::new(chat_id)
+            .get_encryption_info(&ctx)
+            .await
+            .map(|s| s.strdup())
+            .unwrap_or_else(|e| {
+                error!(&ctx, "{}", e);
+                ptr::null_mut()
+            })
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_chat_ephemeral_timer(
     context: *mut dc_context_t,
@@ -1386,7 +1450,28 @@ pub unsafe extern "C" fn dc_get_msg_info(
     }
     let ctx = &*context;
 
-    block_on(message::get_msg_info(&ctx, MsgId::new(msg_id))).strdup()
+    block_on(async move {
+        message::get_msg_info(&ctx, MsgId::new(msg_id))
+            .await
+            .unwrap_or_log_default(ctx, "failed to get msg id")
+            .strdup()
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_msg_html(
+    context: *mut dc_context_t,
+    msg_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_msg_html()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    block_on(MsgId::new(msg_id).get_html(&ctx))
+        .unwrap_or_log_default(ctx, "Failed get_msg_html")
+        .strdup()
 }
 
 #[no_mangle]
@@ -1401,10 +1486,13 @@ pub unsafe extern "C" fn dc_get_mime_headers(
     let ctx = &*context;
 
     block_on(async move {
-        message::get_mime_headers(&ctx, MsgId::new(msg_id))
+        let mime = message::get_mime_headers(&ctx, MsgId::new(msg_id))
             .await
-            .map(|s| s.strdup())
-            .unwrap_or_else(ptr::null_mut)
+            .unwrap_or_log_default(ctx, "failed to get mime headers");
+        if mime.is_empty() {
+            return ptr::null_mut();
+        }
+        mime.strdup()
     })
 }
 
@@ -1434,7 +1522,7 @@ pub unsafe extern "C" fn dc_forward_msgs(
     if context.is_null()
         || msg_ids.is_null()
         || msg_cnt <= 0
-        || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL as u32
+        || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL.to_u32()
     {
         eprintln!("ignoring careless call to dc_forward_msgs()");
         return;
@@ -1473,7 +1561,9 @@ pub unsafe extern "C" fn dc_markseen_msgs(
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ctx = &*context;
 
-    block_on(message::markseen_msgs(&ctx, msg_ids));
+    block_on(message::markseen_msgs(&ctx, msg_ids))
+        .log_err(ctx, "failed dc_markseen_msgs() call")
+        .ok();
 }
 
 #[no_mangle]
@@ -1530,11 +1620,12 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ctx = &*context;
 
-    block_on(Contact::lookup_id_by_addr(
-        &ctx,
-        to_string_lossy(addr),
-        Origin::IncomingReplyTo,
-    ))
+    block_on(async move {
+        Contact::lookup_id_by_addr(&ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
+            .await
+            .unwrap_or_log_default(ctx, "failed to lookup id")
+            .unwrap_or(0)
+    })
 }
 
 #[no_mangle]
@@ -1551,10 +1642,9 @@ pub unsafe extern "C" fn dc_create_contact(
     let name = to_string_lossy(name);
 
     block_on(async move {
-        match Contact::create(&ctx, name, to_string_lossy(addr)).await {
-            Ok(id) => id,
-            Err(_) => 0,
-        }
+        Contact::create(&ctx, &name, &to_string_lossy(addr))
+            .await
+            .unwrap_or(0)
     })
 }
 
@@ -1570,7 +1660,7 @@ pub unsafe extern "C" fn dc_add_address_book(
     let ctx = &*context;
 
     block_on(async move {
-        match Contact::add_address_book(&ctx, to_string_lossy(addr_book)).await {
+        match Contact::add_address_book(&ctx, &to_string_lossy(addr_book)).await {
             Ok(cnt) => cnt as libc::c_int,
             Err(_) => 0,
         }
@@ -1606,7 +1696,12 @@ pub unsafe extern "C" fn dc_get_blocked_cnt(context: *mut dc_context_t) -> libc:
     }
     let ctx = &*context;
 
-    block_on(Contact::get_blocked_cnt(&ctx)) as libc::c_int
+    block_on(async move {
+        Contact::get_all_blocked(&ctx)
+            .await
+            .unwrap_or_log_default(ctx, "failed to get blocked count")
+            .len() as libc::c_int
+    })
 }
 
 #[no_mangle]
@@ -1621,7 +1716,10 @@ pub unsafe extern "C" fn dc_get_blocked_contacts(
 
     block_on(async move {
         Box::into_raw(Box::new(dc_array_t::from(
-            Contact::get_all_blocked(&ctx).await,
+            Contact::get_all_blocked(&ctx)
+                .await
+                .log_err(&ctx, "Can't get blocked contacts")
+                .unwrap_or_default(),
         )))
     })
 }
@@ -1727,13 +1825,15 @@ pub unsafe extern "C" fn dc_imex(
 
     let ctx = &*context;
 
-    let param1 = to_opt_string_lossy(param1);
-
-    spawn(async move {
-        imex::imex(&ctx, what, param1)
-            .await
-            .log_err(ctx, "IMEX failed")
-    });
+    if let Some(param1) = to_opt_string_lossy(param1) {
+        spawn(async move {
+            imex::imex(&ctx, what, param1.as_ref())
+                .await
+                .log_err(ctx, "IMEX failed")
+        });
+    } else {
+        eprintln!("dc_imex called without a valid directory");
+    }
 }
 
 #[no_mangle]
@@ -1748,7 +1848,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
     let ctx = &*context;
 
     block_on(async move {
-        match imex::has_backup(&ctx, to_string_lossy(dir)).await {
+        match imex::has_backup(&ctx, to_string_lossy(dir).as_ref()).await {
             Ok(res) => res.strdup(),
             Err(err) => {
                 // do not bubble up error to the user,
@@ -1829,7 +1929,7 @@ pub unsafe extern "C" fn dc_check_qr(
     let ctx = &*context;
 
     block_on(async move {
-        let lot = qr::check_qr(&ctx, to_string_lossy(qr)).await;
+        let lot = qr::check_qr(&ctx, &to_string_lossy(qr)).await;
         Box::into_raw(Box::new(lot))
     })
 }
@@ -1844,9 +1944,14 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
         return "".strdup();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
     block_on(async move {
-        securejoin::dc_get_securejoin_qr(&ctx, ChatId::new(chat_id))
+        securejoin::dc_get_securejoin_qr(&ctx, chat_id)
             .await
             .unwrap_or_else(|| "".to_string())
             .strdup()
@@ -1879,7 +1984,7 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     chat_id: u32,
     seconds: libc::c_int,
 ) {
-    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL as u32 || seconds < 0 {
+    if context.is_null() || chat_id <= constants::DC_CHAT_ID_LAST_SPECIAL.to_u32() || seconds < 0 {
         eprintln!("ignoring careless call to dc_send_locations_to_chat()");
         return;
     }
@@ -1902,11 +2007,13 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
         return 0;
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
 
-    block_on(location::is_sending_locations_to_chat(
-        &ctx,
-        ChatId::new(chat_id),
-    )) as libc::c_int
+    block_on(location::is_sending_locations_to_chat(&ctx, chat_id)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1938,16 +2045,27 @@ pub unsafe extern "C" fn dc_get_locations(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+    let contact_id = if contact_id == 0 {
+        None
+    } else {
+        Some(contact_id)
+    };
 
     block_on(async move {
         let res = location::get_range(
             &ctx,
-            ChatId::new(chat_id),
+            chat_id,
             contact_id,
             timestamp_begin as i64,
             timestamp_end as i64,
         )
-        .await;
+        .await
+        .unwrap_or_log_default(ctx, "Failed get_locations");
         Box::into_raw(Box::new(dc_array_t::from(res)))
     })
 }
@@ -2328,8 +2446,12 @@ pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut
 
     block_on(async move {
         match ffi_chat.chat.get_profile_image(&ctx).await {
-            Some(p) => p.to_string_lossy().strdup(),
-            None => ptr::null_mut(),
+            Ok(Some(p)) => p.to_string_lossy().strdup(),
+            Ok(None) => ptr::null_mut(),
+            Err(err) => {
+                error!(ctx, "failed to get profile image: {:?}", err);
+                ptr::null_mut()
+            }
         }
     })
 }
@@ -2343,7 +2465,7 @@ pub unsafe extern "C" fn dc_chat_get_color(chat: *mut dc_chat_t) -> u32 {
     let ffi_chat = &*chat;
     let ctx = &*ffi_chat.context;
 
-    block_on(ffi_chat.chat.get_color(&ctx))
+    block_on(ffi_chat.chat.get_color(&ctx)).unwrap_or_log_default(ctx, "Failed get_color")
 }
 
 #[no_mangle]
@@ -2560,6 +2682,16 @@ pub unsafe extern "C" fn dc_msg_get_chat_id(msg: *mut dc_msg_t) -> u32 {
     ffi_msg.message.get_chat_id().to_u32()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_real_chat_id(msg: *mut dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_real_chat_id()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_real_chat_id().to_u32()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_viewtype(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2624,6 +2756,16 @@ pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_cha
     ffi_msg.message.get_text().unwrap_or_default().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_subject(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_subject()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.get_subject().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2722,7 +2864,7 @@ pub unsafe extern "C" fn dc_msg_get_ephemeral_timer(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_ephemeral_timer()
+    ffi_msg.message.get_ephemeral_timer().to_u32()
 }
 
 #[no_mangle]
@@ -2779,6 +2921,17 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_override_sender_name(msg: *mut dc_msg_t) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_override_sender_name()");
+        return "".strdup();
+    }
+    let ffi_msg = &mut *msg;
+
+    ffi_msg.message.get_override_sender_name().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_has_deviating_timestamp(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2859,6 +3012,16 @@ pub unsafe extern "C" fn dc_msg_is_setupmessage(msg: *mut dc_msg_t) -> libc::c_i
     ffi_msg.message.is_setupmessage().into()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_has_html(msg: *mut dc_msg_t) -> libc::c_int {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_has_html()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    ffi_msg.message.has_html().into()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -2874,6 +3037,32 @@ pub unsafe extern "C" fn dc_msg_get_videochat_url(msg: *mut dc_msg_t) -> *mut li
         .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_decide_on_contact_request(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    decision: libc::c_int,
+) -> u32 {
+    if context.is_null() || msg_id <= constants::DC_MSG_ID_LAST_SPECIAL as u32 {
+        eprintln!("ignoring careless call to dc_decide_on_contact_request()");
+    }
+    let ctx = &*context;
+
+    match from_prim(decision) {
+        None => {
+            warn!(ctx, "{} is not a valid decision, ignoring", decision);
+            0
+        }
+        Some(d) => block_on(message::decide_on_contact_request(
+            ctx,
+            MsgId::new(msg_id),
+            d,
+        ))
+        .unwrap_or_default()
+        .to_u32(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_videochat_type(msg: *mut dc_msg_t) -> libc::c_int {
     if msg.is_null() {
@@ -2908,6 +3097,31 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
     ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::c_char) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_html()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.set_html(to_opt_string_lossy(html))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_override_sender_name(
+    msg: *mut dc_msg_t,
+    name: *const libc::c_char,
+) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_override_sender_name()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg
+        .message
+        .set_override_sender_name(to_opt_string_lossy(name))
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_file(
     msg: *mut dc_msg_t,
@@ -3006,6 +3220,11 @@ pub unsafe extern "C" fn dc_msg_set_quote(msg: *mut dc_msg_t, quote: *const dc_m
     let ffi_msg = &mut *msg;
     let ffi_quote = &*quote;
 
+    if ffi_msg.context != ffi_quote.context {
+        eprintln!("ignoring attempt to quote message from a different context");
+        return;
+    }
+
     block_on(async move {
         ffi_msg
             .message
@@ -3107,6 +3326,16 @@ pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut
     ffi_contact.contact.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_auth_name(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_auth_name()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_authname().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_display_name(
     contact: *mut dc_contact_t,
@@ -3131,18 +3360,6 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr(
     ffi_contact.contact.get_name_n_addr().strdup()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_first_name(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
-    if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_first_name()");
-        return "".strdup();
-    }
-    let ffi_contact = &*contact;
-    ffi_contact.contact.get_first_name().strdup()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_profile_image(
     contact: *mut dc_contact_t,
@@ -3159,6 +3376,7 @@ pub unsafe extern "C" fn dc_contact_get_profile_image(
             .contact
             .get_profile_image(&ctx)
             .await
+            .unwrap_or_log_default(ctx, "failed to get profile image")
             .map(|p| p.to_string_lossy().strdup())
             .unwrap_or_else(std::ptr::null_mut)
     })
@@ -3174,6 +3392,16 @@ pub unsafe extern "C" fn dc_contact_get_color(contact: *mut dc_contact_t) -> u32
     ffi_contact.contact.get_color()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_status(contact: *mut dc_contact_t) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_status()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.get_status().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -3282,15 +3510,11 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
 }
 
 trait ResultExt<T, E> {
+    /// Like `log_err()`, but:
+    /// - returns the default value instead of an Err value.
+    /// - emits an error instead of a warning for an [Err] result. This means
+    /// that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
-
-    /// Log a warning to a [ContextWrapper] for an [Err] result.
-    ///
-    /// Does nothing for an [Ok].
-    ///
-    /// You can do this as soon as the wrapper exists, it does not
-    /// have to be open (which is required for the `warn!()` macro).
-    fn log_err(self, wrapper: &Context, message: &str) -> Result<T, E>;
 }
 
 impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
@@ -3303,14 +3527,6 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
             }
         }
     }
-
-    fn log_err(self, ctx: &Context, message: &str) -> Result<T, E> {
-        self.map_err(|err| {
-            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-            warn!(ctx, "{}: {:#}", message, err);
-            err
-        })
-    }
 }
 
 trait ResultNullableExt<T> {
@@ -3351,7 +3567,7 @@ pub unsafe extern "C" fn dc_provider_new_from_email(
         return ptr::null();
     }
     let addr = to_string_lossy(addr);
-    match provider::get_provider_info(addr.as_str()) {
+    match block_on(provider::get_provider_info(addr.as_str())) {
         Some(provider) => provider,
         None => ptr::null_mut(),
     }
@@ -3392,6 +3608,7 @@ pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t)
 }
 
 #[no_mangle]
+#[allow(clippy::needless_return)]
 pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     if provider.is_null() {
         eprintln!("ignoring careless call to dc_provider_unref()");

deltachat-ffi/src/string.rs

@@ -241,7 +241,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] silce can not outlive
+/// Because this returns a reference the [Path] slice can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.13"
+syn = "1.0.72"
 quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -35,7 +35,11 @@ pub fn from_sql_derive(input: TokenStream) -> TokenStream {
         impl rusqlite::types::FromSql for #name {
             fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
                 let inner = rusqlite::types::FromSql::column_result(col)?;
-                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
+                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
+                    Ok(value)
+                } else {
+                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
+                }
             }
         }
     };

draft/group-sync.rst

@@ -5,69 +5,95 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-- Alice and Bob are in a two-member group
+1. Alice and Bob are in a two-member group
 
-- Alice adds Carol, concurrently Bob adds Doris
+2. Then Alice adds Carol, while concurrently Bob adds Doris
 
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
+Right now, the group has inconsistent memberships:
 
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
+- Alice and Carol see a (Alice, Carol, Bob) group
+
+- Bob and Doris see a (Bob, Doris, Alice)
+
+This then leads to "sender is unknown" messages in the chat,
+for example when Alice receives a message from Doris,
+or when Bob receives a message from Carol.
+
+There are also other sources for group membership inconsistency:
+
+- leaving/deleting/adding in larger groups, while being offline,
+  increases chances for inconsistent group membership
+
+- dropped group-membership messages
+
+- group-membership messages landing in "Spam"
 
 
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
+Note that all these problems (can) also happen with verified groups,
+then raising "false alarms" which could lure people to ignore such issues.
 
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
+IOW, it's clear we need to do something about it to improve overall
+reliability in group-settings.
 
 
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
 
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
+Solution: replay group modification messages on inconsistencies
+------------------------------------------------------------------
 
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
+For brevity let's abbreviate "group membership modification" as **GMM**.
 
-Notes:
+Delta chat has explicit GMM messages, typically encrypted to the group members
+as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
 
-- mechanism works for both encrypted and unencrypted add/del messages
+If we detect membership inconsistencies we can resend relevant GMM messages
+to the respective chat.  The receiving devices can process those GMM messages
+as if it would be an incoming message. If for example they have already seen
+the Message-ID of the GMM message, they will ignore the message. It's
+probably useful to record GMM message in their original MIME-format and
+not invent a new recording format. Few notes on three aspects:
 
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
+- **group-membership-tracking**: All valid GMM messages are persisted in
+  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
+  already are in the msgs table, and there is a mime_header column which we could
+  extend to contain the raw Mime GMM message.
 
-- multiple member-added/removed messages can be attached in a single
-  correction message
+- **consistency_checking**: If an incoming GMM has a member list which is
+  not consistent with our own view, broadcast a "Group-Member-Correction"
+  message to all members containing a multipart list of GMMs.
 
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
+- **correcting_memberships**: Upon receiving a Group-Member-Correction
+  message we pass the contained GMMs to the "incoming mail pipeline"
+  (without **consistency_checking** them, to avoid recursion issues)
 
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
 
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
+Alice/Carol and Bob/Doris getting on the same page
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Recall that Alice/Carol and Bob/Doris had a differening view of
+group membership. With the proposed solution, when Bob receives
+Alice's "Carol added" message, he will notice that Alice (and thus
+also carol) did not know about Doris.  Bob's device sends a
+"Chat-Group-Member-Correction" message containing his own GMM
+when adding Doris. Therefore, the group's membership is healed
+for everyone in a single broadcast message.
+
+Alice might also send a Group-member-Correction message,
+so there is a second chance that the group gets to know all GMMs.
+
+Note, for example, that if for some reason Bobs and Carols provider
+drop GMM messages between them (spam) that Alice and Doris can heal
+it by resending GMM messages whenever they detect them to be out of sync.
 
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
+- instead of acting on GMM messages we could send corrections
+  for any received message that addresses inconsistent group members but
+  a) this could delay group-membership healing
   b) could lead to a lot of members sending corrections
+  c) means we might rely on "To-Addresses" which we also like to strike
+     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -83,44 +109,3 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/cmdline.rs

@@ -2,9 +2,11 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
+use deltachat::chat::{
+    self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
+};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -12,15 +14,19 @@ use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
 use deltachat::error::Error;
+use deltachat::export_chat::export_chat_to_zip;
 use deltachat::imex::*;
 use deltachat::location;
+use deltachat::log::LogExt;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::EventType;
 use deltachat::{config, provider};
+use std::fs;
+use std::time::{Duration, SystemTime};
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -116,11 +122,11 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         real_spec = spec.to_string();
         context
             .sql()
-            .set_raw_config(context, "import_spec", Some(&real_spec))
+            .set_raw_config("import_spec", Some(&real_spec))
             .await
             .unwrap();
     } else {
-        let rs = context.sql().get_raw_config(context, "import_spec").await;
+        let rs = context.sql().get_raw_config("import_spec").await.unwrap();
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
             return false;
@@ -171,8 +177,11 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let contact = Contact::get_by_id(context, msg.get_from_id())
         .await
         .expect("invalid contact");
-
-    let contact_name = contact.get_name();
+    let contact_name = if let Some(name) = msg.get_override_sender_name() {
+        format!("~{}", name)
+    } else {
+        contact.get_display_name().to_string()
+    };
     let contact_id = contact.get_id();
 
     let statestr = match msg.get_state() {
@@ -185,7 +194,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -193,7 +202,8 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.get_from_id() == 1 as libc::c_uint {
+        if msg.has_html() { "[HAS-HTML]️" } else { "" },
+        if msg.get_from_id() == 1 {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -251,15 +261,11 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
 }
 
 async fn log_contactlist(context: &Context, contacts: &[u32]) {
-    let mut contacts = contacts.to_vec();
-    if !contacts.contains(&1) {
-        contacts.push(1);
-    }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
-            let name = contact.get_name();
+        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
+            let name = contact.get_display_name();
             let addr = contact.get_addr();
             let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
@@ -288,14 +294,14 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
             let peerstate = Peerstate::from_addr(context, &addr)
                 .await
                 .expect("peerstate error");
-            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
+            if peerstate.is_some() && *contact_id != 1 {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
                 );
             }
 
-            println!("Contact#{}: {}{}", contact_id, line, line2);
+            println!("Contact#{}: {}{}", *contact_id, line, line2);
         }
     }
 }
@@ -355,7 +361,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listarchived\n\
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
-                 createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
@@ -368,9 +373,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
-                 send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -379,12 +384,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 mute <chat-id> [<seconds>]\n\
+                 unmute <chat-id>\n\
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
+                 export-chat <chat-id> <destination-file>\n\
+                 ===========================Contact requests==\n\
+                 decidestartchat <msg-id>\n\
+                 decideblock <msg-id>\n\
+                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -396,10 +409,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  contactinfo <contact-id>\n\
                  delcontact <contact-id>\n\
                  cleanupcontacts\n\
+                 block <contact-id>\n\
+                 unblock <contact-id>\n\
+                 listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
+                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
@@ -443,20 +460,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -500,7 +517,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(&context).await;
+            sql::housekeeping(&context).await.ok_or_log(&context);
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
@@ -512,9 +529,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = std::time::SystemTime::now()
-                .duration_since(time_start)
-                .unwrap_or_default();
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -525,11 +540,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}",
+                        "{}#{}: {} [{} fresh] {}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(&context).await,
+                        chat.get_id().get_fresh_msg_cnt(&context).await?,
+                        if chat.is_muted() { "🔇" } else { "" },
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
@@ -570,7 +586,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -590,7 +606,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let time_start = std::time::SystemTime::now();
+            let msglist =
+                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
+
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -600,27 +620,30 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 })
                 .collect();
 
-            let members = chat::get_chat_contacts(&context, sel_chat.id).await;
+            let members = chat::get_chat_contacts(&context, sel_chat.id).await?;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
                 let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
+            } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
+                "mailinglist".to_string()
             } else {
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{} {}",
+                "{}#{}: {} [{}]{}{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
                 subtitle,
+                if sel_chat.is_muted() { "🔇" } else { "" },
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
-                match sel_chat.get_profile_image(&context).await {
+                match sel_chat.get_profile_image(&context).await? {
                     Some(icon) => match icon.to_str() {
                         Some(icon) => format!(" Icon: {}", icon),
                         _ => " Icon: Err".to_string(),
@@ -640,24 +663,53 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!(
                 "{} messages.",
-                sel_chat.get_id().get_msg_cnt(&context).await
+                sel_chat.get_id().get_msg_cnt(&context).await?
             );
+
+            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
+            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
+
+            println!(
+                "{:?} to create this list, {:?} to mark all messages as noticed.",
+                time_needed, time_noticed_needed
+            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: libc::c_int = arg1.parse()?;
-            let chat_id = chat::create_by_contact_id(&context, contact_id as u32).await?;
+            let contact_id: u32 = arg1.parse()?;
+            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
-        "createchatbymsg" => {
+        "decidestartchat" | "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
-            let chat = Chat::load_from_db(&context, chat_id).await?;
-
-            println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
+            match message::decide_on_contact_request(
+                &context,
+                msg_id,
+                ContactRequestDecision::StartChat,
+            )
+            .await
+            {
+                Some(chat_id) => {
+                    let chat = Chat::load_from_db(&context, chat_id).await?;
+                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
+                }
+                None => println!("Cannot crate chat."),
+            }
+        }
+        "decidenotnow" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
+                .await;
+        }
+        "decideblock" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
+                .await;
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
@@ -677,11 +729,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: libc::c_int = arg1.parse()?;
+            let contact_id_0: u32 = arg1.parse()?;
             if chat::add_contact_to_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_0 as u32,
+                contact_id_0,
             )
             .await
             {
@@ -693,11 +745,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: libc::c_int = arg1.parse()?;
+            let contact_id_1: u32 = arg1.parse()?;
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_1 as u32,
+                contact_id_1,
             )
             .await?;
 
@@ -723,7 +775,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts =
-                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await;
+                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
             log_contactlist(&context, &contacts).await;
@@ -732,7 +784,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    sel_chat.as_ref().unwrap().get_id()
+                    Some(sel_chat.as_ref().unwrap().get_id())
                 )
                 .await,
             );
@@ -740,15 +792,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id = arg1.parse().unwrap_or_default();
+            let contact_id: Option<u32> = arg1.parse().ok();
             let locations = location::get_range(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                Some(sel_chat.as_ref().unwrap().get_id()),
                 contact_id,
                 0,
                 0,
             )
-            .await;
+            .await?;
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
@@ -831,6 +883,22 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "sendhtml" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            ensure!(!arg1.is_empty(), "No html-file given.");
+            let path: &Path = arg1.as_ref();
+            let html = &*fs::read(&path)?;
+            let html = String::from_utf8_lossy(html);
+
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_html(Some(html.to_string()));
+            msg.set_text(Some(if arg2.is_empty() {
+                path.file_name().unwrap().to_string_lossy().to_string()
+            } else {
+                arg2.to_string()
+            }));
+            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
+        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -839,15 +907,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                sel_chat.get_id()
+                Some(sel_chat.get_id())
             } else {
-                ChatId::new(0)
+                None
             };
 
-            let msglist = context.search_msgs(chat, arg1).await;
+            let time_start = std::time::SystemTime::now();
+            let msglist = context.search_msgs(chat, arg1).await?;
+            let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
+            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -860,7 +931,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, Some(&mut draft))
-                    .await;
+                    .await?;
                 println!("Draft saved.");
             } else {
                 sel_chat
@@ -868,7 +939,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, None)
-                    .await;
+                    .await?;
                 println!("Draft deleted.");
             }
         }
@@ -891,7 +962,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Viewtype::Gif,
                 Viewtype::Video,
             )
-            .await;
+            .await?;
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
@@ -917,6 +988,24 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 )
                 .await?;
         }
+        "mute" | "unmute" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            let duration = match arg0 {
+                "mute" => {
+                    if arg2.is_empty() {
+                        MuteDuration::Forever
+                    } else {
+                        SystemTime::now()
+                            .checked_add(Duration::from_secs(arg2.parse()?))
+                            .map_or(MuteDuration::Forever, MuteDuration::Until)
+                    }
+                }
+                "unmute" => MuteDuration::NotMuted,
+                _ => unreachable!("arg0={:?}", arg0),
+            };
+            chat::set_muted(&context, chat_id, duration).await?;
+        }
         "protect" | "unprotect" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
@@ -936,14 +1025,31 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
+        "export-chat" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            ensure!(!arg2.is_empty(), "Argument <destination file> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            // todo check if path is valid (dest dir exists) and ends in .zip
+            export_chat_to_zip(&context, chat_id, arg2).await;
+        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            let res = message::get_msg_info(&context, id).await;
+            let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
+        "html" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let file = dirs::home_dir()
+                .unwrap_or_default()
+                .join(format!("msg-{}.html", id.to_u32()));
+            let html = id.get_html(&context).await?.unwrap_or_default();
+            fs::write(&file, html)?;
+            println!("HTML written to: {:#?}", file);
+        }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs().await;
+            let msglist = context.get_fresh_msgs().await?;
 
             log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
@@ -963,7 +1069,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await;
+            message::markseen_msgs(&context, msg_ids).await?;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -975,9 +1081,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contacts = Contact::get_all(
                 &context,
                 if arg0 == "listverified" {
-                    0x1 | 0x2
+                    DC_GCL_VERIFIED_ONLY | DC_GCL_ADD_SELF
                 } else {
-                    0x2
+                    DC_GCL_ADD_SELF
                 },
                 Some(arg1),
             )
@@ -990,7 +1096,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, book).await?;
+                Contact::add_address_book(&context, &book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }
@@ -998,14 +1104,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id = arg1.parse()?;
+            let contact_id: u32 = arg1.parse()?;
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
             let mut res = format!(
                 "Contact info for: {}:\nIcon: {}\n",
                 name_n_addr,
-                match contact.get_profile_image(&context).await {
+                match contact.get_profile_image(&context).await? {
                     Some(image) => image.to_str().unwrap().to_string(),
                     None => "NoIcon".to_string(),
                 }
@@ -1035,6 +1141,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             Contact::delete(&context, arg1.parse()?).await?;
         }
+        "block" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::block(&context, contact_id).await;
+        }
+        "unblock" => {
+            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
+            let contact_id = arg1.parse()?;
+            Contact::unblock(&context, contact_id).await;
+        }
+        "listblocked" => {
+            let contacts = Contact::get_all_blocked(&context).await?;
+            log_contactlist(&context, &contacts).await;
+            println!("{} blocked contacts.", contacts.len());
+        }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let res = check_qr(&context, arg1).await;
@@ -1055,7 +1176,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1) {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);
@@ -1079,7 +1200,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r as libc::c_int,
+        //         event, event as usize, r,
         //     );
         // }
         "fileinfo" => {

examples/repl/main.rs

@@ -26,8 +26,9 @@ use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
+use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
 
 mod cmdline;
@@ -168,12 +169,14 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 27] = [
+const CHAT_COMMANDS: [&str; 35] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
-    "createchatbymsg",
+    "decidestartchat",
+    "decideblock",
+    "decidenotnow",
     "creategroup",
     "createverified",
     "addmember",
@@ -188,6 +191,7 @@ const CHAT_COMMANDS: [&str; 27] = [
     "send",
     "sendimage",
     "sendfile",
+    "sendhtml",
     "videochat",
     "draft",
     "listmedia",
@@ -195,25 +199,31 @@ const CHAT_COMMANDS: [&str; 27] = [
     "unarchive",
     "pin",
     "unpin",
+    "mute",
+    "unmute",
+    "protect",
+    "unprotect",
     "delchat",
+    "export-chat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
-    "star",
-    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 6] = [
+const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
+    "block",
+    "unblock",
+    "listblocked",
 ];
 const MISC_COMMANDS: [&str; 10] = [
     "getqr",
@@ -229,7 +239,9 @@ const MISC_COMMANDS: [&str; 10] = [
 ];
 
 impl Hinter for DcHelper {
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
+    type Hint = String;
+
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -251,11 +263,10 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
-        if prompt == PROMPT {
+    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
+        if default {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -276,6 +287,7 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
+impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
@@ -300,7 +312,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let (reader_s, reader_r) = async_std::channel::bounded(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -309,8 +321,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
@@ -323,7 +335,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line));
+                    async_std::task::block_on(reader_s.send(line)).unwrap();
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -382,7 +394,7 @@ async fn handle_cmd(
             ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
                     dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
@@ -400,9 +412,8 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
-            {
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")

examples/simple.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use deltachat::chat;
+use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
+    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);

python/CHANGELOG

@@ -1,3 +1,10 @@
+1.51.0
+------
+
+- adapt python bindings and APIs to core51 release 
+  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
+  for more details on all core changes) 
+
 1.44.0
 ------
 

python/README.rst

@@ -11,10 +11,10 @@ Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
-yourself <sourceinstall>`_.
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -142,7 +142,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -151,9 +151,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps scripts/docker_coredeps
 
-This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -19,8 +19,7 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
     if target == 'release':
-        extra = " -C lto=on -C embed-bitcode=yes"
-        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
+        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/_build.py

@@ -9,8 +9,6 @@ import subprocess
 import tempfile
 import textwrap
 import types
-from os.path import abspath
-from os.path import dirname as dn
 
 import cffi
 
@@ -50,6 +48,7 @@ def system_build_flags():
     flags.objs = []
     flags.incs = []
     flags.extra_link_args = []
+    return flags
 
 
 def extract_functions(flags):
@@ -145,9 +144,12 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
+                | DC_GCM
+                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
+                | DC_IMEX
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -165,11 +167,8 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
+        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()

python/src/deltachat/account.py

@@ -89,6 +89,22 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        log("=============== " + self.get_config("displayname") + " ===============")
+        cursor = 0
+        for name, val in self.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 
@@ -214,6 +230,19 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
+        (name, addr) = self.get_contact_addr_and_name(obj, name)
+        name = as_dc_charpointer(name)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        return Contact(self, contact_id)
+
+    def get_contact(self, obj):
+        if isinstance(obj, Contact):
+            return obj
+        (_, addr) = self.get_contact_addr_and_name(obj)
+        return self.get_contact_by_addr(addr)
+
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -229,13 +258,7 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
-        addr = as_dc_charpointer(addr)
-        name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
+        return (name, addr)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -405,23 +428,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=1)
+        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account does not have to be started.
+        Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, 11)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -431,7 +454,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=2)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -439,21 +462,22 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=12)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            self.imex(path, imex_cmd)
             imex_tracker.wait_finish()
 
+    def imex(self, path, imex_cmd):
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self.is_started():
-            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -572,12 +596,34 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def configure(self):
+    def maybe_network(self):
+        """This function should be called when there is a hint
+        that the network is available again,
+        e.g. as a response to system event reporting network availability.
+        The library will try to send pending messages out immediately.
+
+        Moreover, to have a reliable state
+        when the app comes to foreground with network available,
+        it may be reasonable to call the function also at that moment.
+
+        It is okay to call the function unconditionally when there is
+        network available, however, calling the function
+        _without_ having network may interfere with the backoff algorithm
+        and will led to let the jobs fail faster, with fewer retries
+        and may avoid messages being sent out.
+
+        Finally, if the context was created by the dc_accounts_t account manager
+        (currently not implemented in the Python bindings),
+        use dc_accounts_maybe_network() instead of this function
+        """
+        lib.dc_maybe_network(self._dc_context)
+
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert not self.is_configured()
+        assert self.is_configured() == reconfigure
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -585,9 +631,6 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def is_started(self):
-        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
-
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
@@ -597,11 +640,8 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        if bool(lib.dc_is_io_running(self._dc_context)):
-            self.log("dc_stop_io (stop core IO scheduler)")
-            lib.dc_stop_io(self._dc_context)
-        else:
-            self.log("stop_scheduler called on non-running context")
+        self.log("dc_stop_io (stop core IO scheduler)")
+        lib.dc_stop_io(self._dc_context)
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove

python/src/deltachat/chat.py

@@ -167,6 +167,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
+    def get_encryption_info(self):
+        """Return encryption info for this chat.
+
+        :returns: a string with encryption preferences of all chat members"""
+        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
+        return from_dc_charpointer(res)
+
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -247,17 +254,19 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ create a new prepared message.
+        """ prepare a message for sending.
 
         :param msg: the message to be prepared.
-        :returns: :class:`deltachat.message.Message` instance.
+        :returns: a :class:`deltachat.message.Message` instance.
+           This is the same object that was passed in, which
+           has been modified with the new state of the core.
         """
         msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # invalidate passed in message which is not safe to use anymore
-        msg._dc_msg = msg.id = None
-        return Message.from_db(self.account, msg_id)
+        # modify message in place to avoid bad state for the caller
+        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
+        return msg
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -371,7 +380,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
+        contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -495,18 +504,23 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
+                ),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+            )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp):
+    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
+        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -77,6 +77,14 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    @property
+    def status(self):
+        """Get contact status.
+
+        :returns: contact status, empty string if it doesn't exist.
+        """
+        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
+
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 

python/src/deltachat/direct_imap.py

@@ -9,7 +9,9 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
+import imaplib
 import deltachat
+from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -24,13 +26,30 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    if not hasattr(account, "direct_imap"):
-        imap = DirectImap(account)
-        if imap.select_config_folder("mvbox"):
-            imap.delete(ALL, expunge=True)
-        assert imap.select_config_folder("inbox")
-        imap.delete(ALL, expunge=True)
-        setattr(account, "direct_imap", imap)
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
 
 
 @deltachat.global_hookimpl
@@ -50,18 +69,31 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
         host = self.account.get_config("configured_mail_server")
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
+
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
+
+        if security == const.DC_SOCKET_PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.DC_SOCKET_STARTTLS:
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -76,6 +108,12 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
+    def create_folder(self, foldername):
+        try:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
+            print("Can't create", foldername, "probably it already exists:", str(e))
+
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -135,21 +173,6 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()
@@ -226,3 +249,9 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
+
+    def append(self, folder, msg):
+        if msg.startswith("\n"):
+            msg = msg[1:]
+        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(folder, msg)

python/src/deltachat/events.py

@@ -103,6 +103,14 @@ class FFIEventTracker:
             if rex.search(ev.data2):
                 return ev
 
+    def get_info_regex_groups(self, regex, check_error=True):
+        rex = re.compile(regex)
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
+            m = rex.match(ev.data2)
+            if m is not None:
+                return m.groups()
+
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/src/deltachat/message.py

@@ -21,8 +21,8 @@ class Message(object):
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        self.id = lib.dc_msg_get_id(dc_msg)
-        assert self.id is not None and self.id >= 0, repr(self.id)
+        msg_id = self.id
+        assert msg_id is not None and msg_id >= 0, repr(msg_id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
@@ -46,9 +46,13 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is "text", "audio", "video", "file"
+        :param: view_type is the message type code or one of the strings:
+           "text", "audio", "video", "file", "sticker"
         """
-        view_type_code = get_viewtype_code_from_name(view_type)
+        if isinstance(view_type, int):
+            view_type_code = view_type
+        else:
+            view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
@@ -68,6 +72,11 @@ class Message(object):
         self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
         return Chat(self.account, chat_id)
 
+    @props.with_doc
+    def id(self):
+        """id of this message. """
+        return lib.dc_msg_get_id(self._dc_msg)
+
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -77,6 +86,23 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
+    @props.with_doc
+    def html(self):
+        """html text of this messages (might be empty if not an html message). """
+        return from_dc_charpointer(
+            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
+
+    def has_html(self):
+        """return True if this message has an html part, False otherwise."""
+        return lib.dc_msg_has_html(self._dc_msg)
+
+    def set_html(self, html_text):
+        """set the html part of this message.
+
+        It is possible to have text and html part at the same time.
+        """
+        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
+
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -212,6 +238,11 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
+    @property
+    def error(self):
+        """Error message"""
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+
     @property
     def chat(self):
         """chat this message was posted in.
@@ -222,6 +253,20 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    @props.with_doc
+    def override_sender_name(self):
+        """the name that should be shown over the message instead of the contact display name.
+
+        Usually used to impersonate someone else.
+        """
+        return from_dc_charpointer(
+            lib.dc_msg_get_override_sender_name(self._dc_msg))
+
+    def set_override_sender_name(self, name):
+        """set different sender name for a message. """
+        lib.dc_msg_set_override_sender_name(
+            self._dc_msg, as_dc_charpointer(name))
+
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -334,6 +379,10 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
+    def is_sticker(self):
+        """ return True if it's a sticker message. """
+        return self._view_type == const.DC_MSG_STICKER
+
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -354,21 +403,22 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    const.DC_MSG_TEXT: 'text',
-    const.DC_MSG_IMAGE: 'image',
-    const.DC_MSG_GIF: 'gif',
-    const.DC_MSG_AUDIO: 'audio',
-    const.DC_MSG_VIDEO: 'video',
-    const.DC_MSG_FILE: 'file'
+    'text': const.DC_MSG_TEXT,
+    'image': const.DC_MSG_IMAGE,
+    'gif': const.DC_MSG_GIF,
+    'audio': const.DC_MSG_AUDIO,
+    'video': const.DC_MSG_VIDEO,
+    'file': const.DC_MSG_FILE,
+    'sticker': const.DC_MSG_STICKER,
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    for code, value in _view_type_mapping.items():
-        if value == view_type_name:
-            return code
+    code = _view_type_mapping.get(view_type_name)
+    if code is not None:
+        return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
 
 
 #

python/src/deltachat/testplugin.py

@@ -312,24 +312,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1])
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io([ac1, ac2])
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io()
+            self.wait_configure_and_start_io(accounts)
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
+
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
@@ -349,23 +356,29 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self):
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
             started_accounts = []
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    acc._evtracker.consume_events()
-                    acc.get_device_chat().mark_noticed()
-                    del acc._configtracker
-                acc.set_config("bcc_self", "0")
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                    started_accounts.append(acc)
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
             for acc in started_accounts:
                 acc._evtracker.wait_all_initial_fetches()
 
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
@@ -401,13 +414,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
-                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_accepted_chat(self, ac1, ac2):
@@ -503,6 +516,9 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
+        def indent(self, msg):
+            print("  " + msg)
+
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,6 +20,16 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
+    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, int) and ev >= target_progress:
+                assert ev <= progress_upper_limit, \
+                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                return ev
+            if ev == 0:
+                return None
+
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/auditwheels.py

@@ -1,15 +1,17 @@
 
 import os
-import sys
+import platform
 import subprocess
+import sys
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
+    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_x86_64"])
+                 "--plat", "manylinux2014_" + arch])

python/tests/test_account.py

@@ -6,7 +6,10 @@ import queue
 import time
 from deltachat import const, Account
 from deltachat.message import Message
+from deltachat.tracker import ImexTracker
 from deltachat.hookspec import account_hookimpl
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
 from datetime import datetime, timedelta
 
 
@@ -288,6 +291,28 @@ class TestOfflineChat:
         qr = chat.get_join_qr()
         assert ac2.check_qr(qr).is_ask_verifygroup
 
+    def test_removing_blocked_user_from_group(self, ac1, lp):
+        """
+        Test that blocked contact is not unblocked when removed from a group.
+        See https://github.com/deltachat/deltachat-core-rust/issues/2030
+        """
+        lp.sec("Create a group chat with a contact")
+        contact = ac1.create_contact("some1@example.org")
+        group = ac1.create_group_chat("title", contacts=[contact])
+        group.send_text("First group message")
+
+        lp.sec("ac1 blocks contact")
+        contact.block()
+        assert contact.is_blocked()
+
+        lp.sec("ac1 removes contact from their group")
+        group.remove_contact(contact)
+        assert contact.is_blocked()
+
+        lp.sec("ac1 adding blocked contact unblocks it")
+        group.add_contact(contact)
+        assert not contact.is_blocked()
+
     def test_get_set_profile_image_simple(self, ac1, data):
         chat = ac1.create_group_chat(name="title1")
         p = data.get_path("d.png")
@@ -410,11 +435,11 @@ class TestOfflineChat:
         email = "hello <hello@example.org>"
         contact1 = ac1.create_contact(email)
         assert contact1.addr == "hello@example.org"
-        assert contact1.display_name == "hello"
+        assert contact1.name == "hello"
         contact1 = ac1.create_contact(email, name="world")
-        assert contact1.display_name == "world"
+        assert contact1.name == "world"
         contact2 = ac1.create_contact("display1 <x@example.org>", "real")
-        assert contact2.display_name == "real"
+        assert contact2.name == "real"
 
     def test_create_chat_simple(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
@@ -451,6 +476,7 @@ class TestOfflineChat:
         contact = msg.get_sender_contact()
         assert contact == ac1.get_self_contact()
         assert not backupdir.listdir()
+        ac1.stop_io()
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
         ac2 = acfactory.get_unconfigured_account()
@@ -465,10 +491,6 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
-    def test_ac_setup_message_fails(self, ac1):
-        with pytest.raises(RuntimeError):
-            ac1.initiate_key_transfer()
-
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -576,6 +598,28 @@ class TestOfflineChat:
         assert in_list[1][1] == chat
         assert in_list[1][2] == contacts[3]
 
+    def test_audit_log_view_without_daymarker(self, ac1, lp):
+        lp.sec("ac1: test audit log (show only system messages)")
+        chat = ac1.create_group_chat(name="audit log sample data")
+        # promote chat
+        chat.send_text("hello")
+        assert chat.is_promoted()
+
+        lp.sec("create test data")
+        chat.add_contact(ac1.create_contact("some-1@example.org"))
+        chat.set_name("audit log test group")
+        chat.send_text("a message in between")
+
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
+        lp.sec("check message count of only system messages (without daymarkers)")
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2
+
 
 def test_basic_imap_api(acfactory, tmpdir):
     ac1, ac2 = acfactory.get_two_online_accounts()
@@ -609,7 +653,7 @@ class TestOnlineAccount:
             config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
         )
         # rsa key gen can be slow especially on CI, adjust timeout
-        ac1._evtracker.set_timeout(120)
+        ac1._evtracker.set_timeout(240)
         acfactory.wait_configure_and_start_io()
         chat = acfactory.get_accepted_chat(ac1, ac2)
 
@@ -646,7 +690,7 @@ class TestOnlineAccount:
         except Exception:
             pass
 
-    def test_export_import_self_keys(self, acfactory, tmpdir):
+    def test_export_import_self_keys(self, acfactory, tmpdir, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         dir = tmpdir.mkdir("exportdir")
@@ -654,8 +698,17 @@ class TestOnlineAccount:
         assert len(export_files) == 2
         for x in export_files:
             assert x.startswith(dir.strpath)
+        key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
         ac1._evtracker.consume_events()
+
+        lp.sec("exported keys (private and public)")
+        for name in os.listdir(dir.strpath):
+            lp.indent(dir.strpath + os.sep + name)
+        lp.sec("importing into existing account")
         ac2.import_self_keys(dir.strpath)
+        key_id2, = ac2._evtracker.get_info_regex_groups(
+            r".*stored.*KeyId\((.*)\).*", check_error=False)
+        assert key_id2 == key_id
 
     def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -763,6 +816,48 @@ class TestOnlineAccount:
         assert open(msg.filename).read() == content
         assert msg.filename.endswith(basename)
 
+    def test_html_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        html_text = "<p>hello HTML world</p>"
+
+        lp.sec("ac1: prepare and send text message to ac2")
+        msg1 = chat.send_text("message0")
+        assert not msg1.has_html()
+        assert msg1.html == ""
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message0"
+        assert not msg2.has_html()
+        assert msg2.html == ""
+
+        lp.sec("ac1: prepare and send HTML+text message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message1")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+        assert msg1.has_html()
+        assert html_text in msg1.html
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
+        lp.sec("ac1: prepare and send HTML-only message to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_html(html_text)
+        msg1 = chat.send_msg(msg1)
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert "<p>" not in msg2.text
+        assert "hello HTML world" in msg2.text
+        assert msg2.has_html()
+        assert html_text in msg2.html
+
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, move=True, sentbox=True)
@@ -938,6 +1033,116 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_moved_markseen(self, acfactory, lp):
+        """Test that message already moved to DeltaChat folder is marked as seen."""
+        ac1 = acfactory.get_online_configuring_account(mvbox=True, config={"inbox_watch": "0"})
+        ac2 = acfactory.get_online_configuring_account()
+        acfactory.wait_configure_and_start_io([ac1, ac2])
+        ac1.set_config("bcc_self", "1")
+
+        ac1.direct_imap.idle_start()
+        ac1.create_chat(ac2).send_text("Hello!")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.stop_io()
+
+        # Emulate moving of the message to DeltaChat folder by Sieve rule.
+        # mailcow server contains this rule by default.
+        ac1.direct_imap.conn.move(["*"], "DeltaChat")
+
+        ac1.direct_imap.select_folder("DeltaChat")
+        ac1.direct_imap.idle_start()
+        ac1.start_io()
+        ac1.direct_imap.idle_wait_for_seen()
+        ac1.direct_imap.idle_done()
+
+        fetch = list(ac1.direct_imap.conn.fetch("*", b'FLAGS').values())
+        flags = fetch[-1][b'FLAGS']
+        is_seen = b'\\Seen' in flags
+        assert is_seen
+
+    def test_message_override_sender_name(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = acfactory.get_accepted_chat(ac1, ac2)
+        overridden_name = "someone else"
+
+        ac1.set_config("displayname", "ac1")
+
+        lp.sec("sending text message with overridden name from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_override_sender_name(overridden_name)
+        msg1.set_text("message1")
+        msg1 = chat.send_msg(msg1)
+        assert msg1.override_sender_name == overridden_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message1"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert msg2.override_sender_name == overridden_name
+
+        lp.sec("sending normal text message from ac1 to ac2")
+        msg1 = Message.new_empty(ac1, "text")
+        msg1.set_text("message2")
+        msg1 = chat.send_msg(msg1)
+        assert not msg1.override_sender_name
+
+        lp.sec("wait for ac2 to receive message")
+        msg2 = ac2._evtracker.wait_next_incoming_message()
+        assert msg2.text == "message2"
+        assert msg2.get_sender_contact().name == ac1.get_config("displayname")
+        assert not msg2.override_sender_name
+
+    @pytest.mark.parametrize("mvbox_move", [True, False])
+    def test_markseen_message_and_mdn(self, acfactory, mvbox_move):
+        # Please only change this test if you are very sure that it will still catch the issues it catches now.
+        # We had so many problems with markseen, if in doubt, rather create another test, it can't harm.
+        ac1 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+        ac2 = acfactory.get_online_configuring_account(move=mvbox_move, mvbox=mvbox_move)
+
+        acfactory.wait_configure_and_start_io()
+        # Do not send BCC to self, we only want to test MDN on ac1.
+        ac1.set_config("bcc_self", "0")
+
+        folder = "mvbox" if mvbox_move else "inbox"
+        ac1.direct_imap.select_config_folder(folder)
+        ac2.direct_imap.select_config_folder(folder)
+        ac1.direct_imap.idle_start()
+        ac2.direct_imap.idle_start()
+
+        acfactory.get_accepted_chat(ac1, ac2).send_text("hi")
+        msg = ac2._evtracker.wait_next_incoming_message()
+
+        ac2.mark_seen_messages([msg])
+
+        ac1.direct_imap.idle_wait_for_seen()  # Check that the mdn is marked as seen
+        ac2.direct_imap.idle_wait_for_seen()  # Check that the original message is marked as seen
+        ac1.direct_imap.idle_done()
+        ac2.direct_imap.idle_done()
+
+    def test_reply_privately(self, acfactory):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        group1 = ac1.create_group_chat("group")
+        group1.add_contact(ac2)
+        group1.send_text("hello")
+
+        msg2 = ac2._evtracker.wait_next_messages_changed()
+        group2 = msg2.create_chat()
+        assert group2.get_name() == group1.get_name()
+
+        msg_reply = Message.new_empty(ac2, "text")
+        msg_reply.set_text("message reply")
+        msg_reply.quote = msg2
+
+        private_chat1 = ac1.create_chat(ac2)
+        private_chat2 = ac2.create_chat(ac1)
+        private_chat2.send_msg(msg_reply)
+
+        msg_reply1 = ac1._evtracker.wait_next_incoming_message()
+        assert msg_reply1.quoted_text == "hello"
+        assert not msg_reply1.chat.is_group()
+        assert msg_reply1.chat.id == private_chat1.id
+
     def test_mdn_asymetric(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts(move=True)
 
@@ -1017,6 +1222,64 @@ class TestOnlineAccount:
         assert not msg.is_encrypted()
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    def test_gossip_encryption_preference(self, acfactory, lp):
+        """Test that encryption preference of group members is gossiped to new members.
+        This is a Delta Chat extension to Autocrypt 1.1.0, which Autocrypt-Gossip headers
+        SHOULD NOT contain encryption preference.
+        """
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
+
+        lp.sec("ac1 learns that ac2 prefers encryption")
+        ac1.create_chat(ac2)
+        msg = ac2.create_chat(ac1).send_text("first message")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "first message"
+        assert not msg.is_encrypted()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config('addr'))
+        assert msg.chat.get_encryption_info() == res
+        lp.sec("ac2 learns that ac3 prefers encryption")
+        ac2.create_chat(ac3)
+        msg = ac3.create_chat(ac2).send_text("I prefer encryption")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "I prefer encryption"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac3 does not know that ac1 prefers encryption")
+        ac1.create_chat(ac3)
+        chat = ac3.create_chat(ac1)
+        res = "{} No encryption.".format(ac1.get_config('addr'))
+        assert chat.get_encryption_info() == res
+        msg = chat.send_text("not encrypted")
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "not encrypted"
+        assert not msg.is_encrypted()
+
+        lp.sec("ac1 creates a group chat with ac2")
+        group_chat = ac1.create_group_chat("hello")
+        group_chat.add_contact(ac2)
+        encryption_info = group_chat.get_encryption_info()
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert encryption_info == res
+        msg = group_chat.send_text("hi")
+
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.text == "hi"
+
+        lp.sec("ac2 adds ac3 to the group")
+        msg.chat.add_contact(ac3)
+        assert msg.is_encrypted()
+
+        lp.sec("ac3 learns that ac1 prefers encryption")
+        msg = ac3._evtracker.wait_next_incoming_message()
+        encryption_info = msg.chat.get_encryption_info().splitlines()
+        res = "{} End-to-end encryption preferred.".format(ac1.get_config("addr"))
+        assert res in encryption_info
+        res = "{} End-to-end encryption preferred.".format(ac2.get_config("addr"))
+        assert res in encryption_info
+        msg = chat.send_text("encrypted")
+        assert msg.is_encrypted()
+
     def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac2.set_config("save_mime_headers", "1")
@@ -1056,6 +1319,52 @@ class TestOnlineAccount:
         assert not device_chat.can_send()
         assert device_chat.get_draft() is None
 
+    def test_dont_show_emails_in_draft_folder(self, acfactory):
+        """Most mailboxes have a "Drafts" folder where constantly new emails appear but we don't actually want to show them.
+        So: If it's outgoing AND there is no Received header AND it's not in the sentbox, then ignore the email."""
+        ac1 = acfactory.get_online_configuring_account()
+        ac1.set_config("show_emails", "2")
+        ac1.create_contact("alice@example.com").create_chat()
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder("Drafts")
+        ac1.direct_imap.create_folder("Sent")
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing again:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        ac1.stop_io()
+
+        ac1.direct_imap.append("Drafts", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Drafts
+        """.format(ac1.get_config("configured_addr")))
+        ac1.direct_imap.append("Sent", """
+            From: ac1 <{}>
+            Subject: subj
+            To: alice@example.com
+            Message-ID: <hsabaeni@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            message in Sent
+        """.format(ac1.get_config("configured_addr")))
+
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+
+        msg = ac1._evtracker.wait_next_messages_changed()
+
+        # Wait until each folder was scanned, this is necessary for this test to test what it should test:
+        ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+
+        assert msg.text == "subj – message in Sent"
+        assert len(msg.chat.get_messages()) == 1
+
     def test_prefer_encrypt(self, acfactory, lp):
         """Test quorum rule for encryption preference in 1:1 and group chat."""
         ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
@@ -1106,8 +1415,8 @@ class TestOnlineAccount:
         # Majority prefers encryption now
         assert msg5.is_encrypted()
 
-    def test_reply_encrypted(self, acfactory, lp):
-        """Test that replies to encrypted messages are encrypted."""
+    def test_quote_encrypted(self, acfactory, lp):
+        """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
 
         lp.sec("ac1: create chat with ac2")
@@ -1156,6 +1465,39 @@ class TestOnlineAccount:
             assert msg_in.quoted_text == quoted_msg.text
             assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
 
+    def test_quote_attachment(self, tmpdir, acfactory, lp):
+        """Test that replies with an attachment and a quote are received correctly."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1 creates chat with ac2")
+        chat1 = ac1.create_chat(ac2)
+
+        lp.sec("ac1 sends text message to ac2")
+        chat1.send_text("hi")
+
+        lp.sec("ac2 receives contact request from ac1")
+        received_message = ac2._evtracker.wait_next_messages_changed()
+        assert received_message.text == "hi"
+
+        basename = "attachment.txt"
+        p = os.path.join(tmpdir.strpath, basename)
+        with open(p, "w") as f:
+            f.write("data to send")
+
+        lp.sec("ac2 sends a reply to ac1")
+        chat2 = received_message.create_chat()
+        reply = Message.new_empty(ac2, "file")
+        reply.set_text("message reply")
+        reply.set_file(p)
+        reply.quote = received_message
+        chat2.send_msg(reply)
+
+        lp.sec("ac1 receives a reply from ac2")
+        received_reply = ac1._evtracker.wait_next_incoming_message()
+        assert received_reply.text == "message reply"
+        assert received_reply.quoted_text == received_message.text
+        assert open(received_reply.filename).read() == "data to send"
+
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -1249,19 +1591,58 @@ class TestOnlineAccount:
         m = message_queue.get()
         assert m == msg_in
 
-    def test_import_export_online_all(self, acfactory, tmpdir, lp):
+    def test_import_export_online_all(self, acfactory, tmpdir, data, lp):
         ac1 = acfactory.get_one_online_account()
 
         lp.sec("create some chat content")
-        contact1 = ac1.create_contact("some1@example.org", name="some1")
-        contact1.create_chat().send_text("msg1")
+        chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+        chat1.send_text("msg1")
         assert len(ac1.get_contacts(query="some1")) == 1
+
+        original_image_path = data.get_path("d.png")
+        chat1.send_image(original_image_path)
+
+        # Add another 100KB file that ensures that the progress is smooth enough
+        path = tmpdir.join("attachment.txt")
+        with open(path, "w") as file:
+            file.truncate(100000)
+        chat1.send_file(path.strpath)
+
+        def assert_account_is_proper(ac):
+            contacts = ac.get_contacts(query="some1")
+            assert len(contacts) == 1
+            contact2 = contacts[0]
+            assert contact2.addr == "some1@example.org"
+            chat2 = contact2.create_chat()
+            messages = chat2.get_messages()
+            assert len(messages) == 3
+            assert messages[0].text == "msg1"
+            assert messages[1].filemime == "image/png"
+            assert os.stat(messages[1].filename).st_size == os.stat(original_image_path).st_size
+            ac.set_config("displayname", "new displayname")
+            assert ac.get_config("displayname") == "new displayname"
+
+        assert_account_is_proper(ac1)
+
         backupdir = tmpdir.mkdir("backup")
 
         lp.sec("export all to {}".format(backupdir))
-        path = ac1.export_all(backupdir.strpath)
-        assert os.path.exists(path)
-        t = time.time()
+        with ac1.temp_plugin(ImexTracker()) as imex_tracker:
+
+            ac1.stop_io()
+            ac1.imex(backupdir.strpath, const.DC_IMEX_EXPORT_BACKUP)
+
+            # check progress events for export
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(250, progress_upper_limit=499)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+
+            paths = imex_tracker.wait_finish()
+            assert len(paths) == 1
+            path = paths[0]
+            assert os.path.exists(path)
+            ac1.start_io()
 
         lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
@@ -1271,22 +1652,20 @@ class TestOnlineAccount:
         assert path2 == path
 
         lp.sec("import backup and check it's proper")
-        ac2.import_all(path)
-        contacts = ac2.get_contacts(query="some1")
-        assert len(contacts) == 1
-        contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
-        chat2 = contact2.create_chat()
-        messages = chat2.get_messages()
-        assert len(messages) == 1
-        assert messages[0].text == "msg1"
+        with ac2.temp_plugin(ImexTracker()) as imex_tracker:
+            ac2.import_all(path)
+
+            # check progress events for import
+            assert imex_tracker.wait_progress(1, progress_upper_limit=249)
+            assert imex_tracker.wait_progress(500, progress_upper_limit=749)
+            assert imex_tracker.wait_progress(750, progress_upper_limit=999)
+            assert imex_tracker.wait_progress(1000)
+
+        assert_account_is_proper(ac1)
+        assert_account_is_proper(ac2)
 
-        # wait until a second passed since last backup
-        # because get_latest_backupfile() shall return the latest backup
-        # from a UI it's unlikely anyone manages to export two
-        # backups in one second.
-        time.sleep(max(0, 1 - (time.time() - t)))
         lp.sec("Second-time export all to {}".format(backupdir))
+        ac1.stop_io()
         path2 = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path2)
         assert path2 != path
@@ -1364,7 +1743,7 @@ class TestOnlineAccount:
         ac1._evtracker.wait_securejoin_inviter_progress(1000)
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1, ac2, ac3 = acfactory.get_many_online_accounts(3)
         lp.sec("ac1: create verified-group QR, ac2 scans and joins")
         chat1 = ac1.create_group_chat("hello", verified=True)
         assert chat1.is_protected()
@@ -1395,6 +1774,29 @@ class TestOnlineAccount:
         assert msg.text == "world"
         assert msg.is_encrypted()
 
+        lp.sec("ac1: create QR code and let ac3 scan it, starting the securejoin")
+        qr = ac1.get_setup_contact_qr()
+
+        lp.sec("ac3: start QR-code based setup contact protocol")
+        ch = ac3.qr_setup_contact(qr)
+        assert ch.id >= 10
+        ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+        lp.sec("ac1: add ac3 to verified group")
+        chat1.add_contact(ac3)
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.is_encrypted()
+        assert msg.is_system_message()
+        assert not msg.error
+
+        lp.sec("ac2: send message and let ac3 read it")
+        chat2.send_text("hi")
+        # Skip system message about added member
+        ac3._evtracker.wait_next_incoming_message()
+        msg = ac3._evtracker.wait_next_incoming_message()
+        assert msg.text == "hi"
+        assert msg.is_encrypted()
+
     def test_set_get_contact_avatar(self, acfactory, data, lp):
         lp.sec("configuring ac1 and ac2")
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1521,7 +1923,7 @@ class TestOnlineAccount:
 
         lp.sec("ac1 blocks ac2")
         contact = ac1.create_contact(ac2)
-        contact.set_blocked()
+        contact.block()
         assert contact.is_blocked()
         ev = ac1._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         assert ev.data1 == contact.id
@@ -1598,6 +2000,47 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
+    def test_fetch_deleted_msg(self, acfactory, lp):
+        """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
+        hundreds of times, because uid_next was not updated.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2429.
+        """
+        ac1 = acfactory.get_one_online_account()
+        ac1.stop_io()
+
+        ac1.direct_imap.append("INBOX", """
+            From: alice <alice@example.org>
+            Subject: subj
+            To: bob@example.com
+            Chat-Version: 1.0
+            Message-ID: <aepiors@example.org>
+            Content-Type: text/plain; charset=utf-8
+
+            Deleted message
+        """)
+        ac1.direct_imap.delete("1:*", expunge=False)
+        ac1.start_io()
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                break
+
+        # The message was downloaded once, now check that it's not downloaded again
+
+        for ev in ac1._evtracker.iter_events():
+            if ev.name == "DC_EVENT_INFO" and "1 mails read from" in ev.data2:
+                pytest.fail("The same email was read twice")
+
+            if ev.name == "DC_EVENT_MSGS_CHANGED":
+                pytest.fail("A deleted message was shown to the user")
+
+            if ev.name == "DC_EVENT_INFO" and "INBOX: Idle entering wait-on-remote state" in ev.data2:
+                break  # DC is done with reading messages
+
     def test_send_receive_locations(self, acfactory, lp):
         now = datetime.utcnow()
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1620,7 +2063,7 @@ class TestOnlineAccount:
 
         ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
         ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
-        chat1.send_text("hello")
+        chat1.send_text("🍞")
         ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
         lp.sec("ac2: wait for incoming location message")
@@ -1634,6 +2077,7 @@ class TestOnlineAccount:
         assert locations[0].longitude == 3.0
         assert locations[0].accuracy == 0.5
         assert locations[0].timestamp > now
+        assert locations[0].marker == "🍞"
 
         contact = ac2.create_contact(ac1)
         locations2 = chat2.get_locations(contact=contact)
@@ -1685,6 +2129,7 @@ class TestOnlineAccount:
         # Error message should be assigned to the chat with ac1.
         lp.sec("ac4: checking that message is assigned to the sender chat")
         error_msg = ac4._evtracker.wait_next_incoming_message()
+        assert error_msg.error  # There is an error decrypting the message
         assert error_msg.chat == chat41
 
         lp.sec("ac2: sending a reply to the chat")
@@ -1695,6 +2140,7 @@ class TestOnlineAccount:
 
         lp.sec("ac4: checking that reply is assigned to ac2 chat")
         error_reply = ac4._evtracker.wait_next_incoming_message()
+        assert error_reply.error  # There is an error decrypting the message
         assert error_reply.chat == chat42
 
         # Test that ac4 replies to error messages don't appear in the
@@ -1706,11 +2152,13 @@ class TestOnlineAccount:
         chat42.send_text("I can't decrypt your message, ac2!")
 
         msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac1!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac1.create_chat(ac3)
 
         msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.error is None
         assert msg.text == "I can't decrypt your message, ac2!"
         assert msg.is_encrypted(), "Message is not encrypted"
         assert msg.chat == ac2.create_chat(ac4)
@@ -1871,7 +2319,9 @@ class TestOnlineAccount:
         ac1, ac2 = acfactory.get_two_online_accounts()
         ac1.set_config("displayname", "Account 1")
 
-        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        # Similar to acfactory.get_accepted_chat, but without setting the contact name.
+        ac2.create_contact(ac1.get_config("addr")).create_chat()
+        chat12 = ac1.create_contact(ac2.get_config("addr")).create_chat()
         contact = None
 
         def update_name():
@@ -1902,12 +2352,40 @@ class TestOnlineAccount:
         # so it should not be changed.
         ac1.set_config("displayname", "Renamed again")
         updated_name = update_name()
-        if updated_name == "Renamed again":
-            # Known bug, mark as XFAIL
-            pytest.xfail("Contact was renamed after explicit rename")
-        else:
-            # No renames should happen after explicit rename
-            assert updated_name == "Renamed"
+        assert updated_name == "Renamed"
+
+    def test_status(self, acfactory):
+        """Test that status is transferred over the network."""
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        chat12 = acfactory.get_accepted_chat(ac1, ac2)
+        ac1.set_config("selfstatus", "New status")
+        chat12.send_text("hi")
+        msg_received = ac2._evtracker.wait_next_incoming_message()
+        assert msg_received.text == "hi"
+        assert msg_received.get_sender_contact().status == "New status"
+
+        # Send a reply from ac2 to ac1 so ac1 can send a read receipt.
+        reply_msg = msg_received.chat.send_text("reply")
+        reply_msg_received = ac1._evtracker.wait_next_incoming_message()
+        assert reply_msg_received.text == "reply"
+
+        # Send read receipt from ac1 to ac2.
+        # It does not contain the signature.
+        ac1.mark_seen_messages([reply_msg_received])
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSG_READ")
+        assert ev.data1 == reply_msg.chat.id
+        assert ev.data2 == reply_msg.id
+        assert reply_msg.is_out_mdn_received()
+
+        # Test that the status is not cleared as a result of receiving a read receipt.
+        assert msg_received.get_sender_contact().status == "New status"
+
+        ac1.set_config("selfstatus", "")
+        chat12.send_text("hello")
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+        assert msg.get_sender_contact().status == ""
 
     def test_group_quote(self, acfactory, lp):
         """Test quoting in a group with a new member who have not seen the quoted message."""
@@ -1948,42 +2426,170 @@ class TestOnlineAccount:
         assert received_reply.quoted_text == "hello"
         assert received_reply.quote.id == out_msg.id
 
+    @pytest.mark.parametrize("folder,move,expected_destination,inbox_watch,", [
+        ("xyz", False, "xyz", "1"),  # Test that emails are recognized in a random folder but not moved
+        ("xyz", True, "DeltaChat", "1"),  # ...emails are found in a random folder and moved to DeltaChat
+        ("Spam", False, "INBOX", "1"),  # ...emails are moved from the spam folder to the Inbox
+        ("INBOX", False, "INBOX", "0"),  # ...emails are found in the `Inbox` folder even if `inbox_watch` is "0"
+    ])
+    # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
+    # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
+    def test_scan_folders(self, acfactory, lp, folder, move, expected_destination, inbox_watch):
+        """Delta Chat periodically scans all folders for new messages to make sure we don't miss any."""
+        variant = folder + "-" + str(move) + "-" + expected_destination
+        lp.sec("Testing variant " + variant)
+        ac1 = acfactory.get_online_configuring_account(move=move)
+        ac2 = acfactory.get_online_configuring_account()
+        ac1.set_config("inbox_watch", inbox_watch)
+
+        acfactory.wait_configure(ac1)
+        ac1.direct_imap.create_folder(folder)
+
+        acfactory.wait_configure_and_start_io()
+        # Wait until each folder was selected once and we are IDLEing:
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        ac1.stop_io()
+
+        # Send a message to ac1 and move it to the mvbox:
+        ac1.direct_imap.select_config_folder("inbox")
+        ac1.direct_imap.idle_start()
+        acfactory.get_accepted_chat(ac2, ac1).send_text("hello")
+        ac1.direct_imap.idle_check(terminate=True)
+        ac1.direct_imap.conn.move(["*"], folder)  # "*" means "biggest UID in mailbox"
+
+        lp.sec("Everything prepared, now see if DeltaChat finds the message (" + variant + ")")
+        ac1.set_config("scan_all_folders_debounce_secs", "0")
+        ac1.start_io()
+        msg = ac1._evtracker.wait_next_incoming_message()
+        assert msg.text == "hello"
+
+        # Wait until the message was moved (if at all) and we are IDLEing again:
+        if inbox_watch == "1":
+            ac1._evtracker.get_info_contains("INBOX: Idle entering wait-on-remote state")
+        else:
+            ac1._evtracker.get_info_contains("IMAP-fake-IDLE: no folder, waiting for interrupt")
+        ac1.direct_imap.select_folder(expected_destination)
+        assert len(ac1.direct_imap.get_all_messages()) == 1
+        if folder != expected_destination:
+            ac1.direct_imap.select_folder(folder)
+            assert len(ac1.direct_imap.get_all_messages()) == 0
+
     @pytest.mark.parametrize("mvbox_move", [False, True])
-    def test_add_all_recipients_as_contacts(self, acfactory, lp, mvbox_move):
+    def test_fetch_existing(self, acfactory, lp, mvbox_move):
         """Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
         This way, we can already offer them some email addresses they can write to.
 
-        Also test that existing emails are fetched during onboarding.
+        Also, the newest existing emails from each folder are fetched during onboarding.
+
+        Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
+
+        def assert_folders_configured(ac):
+            """There was a bug that scan_folders() set the configured folders to None under some circumstances.
+            So, check that they are still configured:"""
+            assert ac.get_config("configured_sentbox_folder") == "Sent"
+            if mvbox_move:
+                assert ac.get_config("configured_mvbox_folder")
 
-        Lastly, tests that bcc_self messages moved to the mvbox are marked as read."""
         ac1 = acfactory.get_online_configuring_account(mvbox=mvbox_move, move=mvbox_move)
+        ac1.set_config("sentbox_move", "1")
         ac2 = acfactory.get_online_configuring_account()
 
+        acfactory.wait_configure(ac1)
+
+        ac1.direct_imap.create_folder("Sent")
+        ac1.set_config("sentbox_watch", "1")
+
+        # We need to reconfigure to find the new "Sent" folder.
+        # `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
+        # would also find the "Sent" folder, but it would be too late:
+        # The sentbox thread, started by `start_io()`, would have seen that there is no
+        # ConfiguredSentboxFolder and do nothing.
+        ac1._configtracker = ac1.configure(reconfigure=True)
         acfactory.wait_configure_and_start_io()
+        assert_folders_configured(ac1)
 
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("send out message with bcc to ourselves")
         if mvbox_move:
             ac1.direct_imap.select_config_folder("mvbox")
+        else:
+            ac1.direct_imap.select_config_folder("sentbox")
         ac1.direct_imap.idle_start()
+
+        lp.sec("send out message with bcc to ourselves")
         ac1.set_config("bcc_self", "1")
+        chat = acfactory.get_accepted_chat(ac1, ac2)
         chat.send_text("message text")
+        assert_folders_configured(ac1)
 
         # now wait until the bcc_self message arrives
         # Also test that bcc_self messages moved to the mvbox are marked as read.
         assert ac1.direct_imap.idle_wait_for_seen()
+        assert_folders_configured(ac1)
 
         ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
         ac1_clone._configtracker.wait_finish()
         ac1_clone.start_io()
+        assert_folders_configured(ac1_clone)
 
         ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
         ac2_addr = ac2.get_config("addr")
         assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
+        assert_folders_configured(ac1_clone)
 
         msg = ac1_clone._evtracker.wait_next_messages_changed()
         assert msg.text == "message text"
+        assert_folders_configured(ac1)
+        assert_folders_configured(ac1_clone)
+
+    def test_fetch_existing_msgs_group_and_single(self, acfactory, lp):
+        """There was a bug concerning fetch-existing-msgs:
+
+        A sent a message to you, adding you to a group. This created a contact request.
+        You wrote a message to A, creating a chat.
+        ...but the group stayed blocked.
+        So, after fetch-existing-msgs you have one contact request and one chat with the same person.
+
+        See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+
+        acfactory.wait_configure_and_start_io()
+
+        lp.sec("receive a message")
+        ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
+        ac1._evtracker.wait_next_messages_changed()
+
+        lp.sec("send out message with bcc to ourselves")
+        ac1.direct_imap.idle_start()
+        ac1.set_config("bcc_self", "1")
+        ac1.create_chat(ac2).send_text("outgoing, encrypted direct message, creating a chat")
+
+        # now wait until the bcc_self message arrives
+        assert ac1.direct_imap.idle_wait_for_seen()
+
+        lp.sec("Clone online account and let it fetch the existing messages")
+        ac1_clone = acfactory.clone_online_account(ac1)
+        ac1_clone.set_config("fetch_existing_msgs", "1")
+        ac1_clone._configtracker.wait_finish()
+
+        ac1_clone.start_io()
+        ac1_clone._evtracker.wait_all_initial_fetches()
+        chats = ac1_clone.get_chats()
+        assert len(chats) == 4  # two newly created chats + self-chat + device-chat
+        group_chat = [c for c in chats if c.get_name() == "group name"][0]
+        assert group_chat.is_group()
+        private_chat = [c for c in chats if c.get_name() == "ac2"][0]
+        assert not private_chat.is_group()
+
+        group_messages = group_chat.get_messages()
+        assert len(group_messages) == 1
+        assert group_messages[0].text == "incoming, unencrypted group message"
+        private_messages = private_chat.get_messages()
+        # We can't decrypt the message in this chat, so the chat is empty:
+        assert len(private_messages) == 0
 
 
 class TestGroupStressTests:

python/tox.ini

@@ -1,7 +1,6 @@
 [tox]
-# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py37
+    py3
     lint
     auditwheels
 
@@ -13,7 +12,6 @@ passenv =
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
-    DCC_PY_LIVECONFIG
     DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
@@ -47,8 +45,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
-    breathe
+    # Pin dependencies to the versions which actually work with Python 3.5.
+    sphinx==3.4.3
+    breathe==4.28.0
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

rust-toolchain

@@ -1 +1 @@
-1.45.0
+1.50.0

scripts/README.md

@@ -1,11 +1,15 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through CircleCI and an own build machine.
+Continuous Integration, run through [GitHub
+Actions](https://docs.github.com/actions),
+[CircleCI](https://app.circleci.com/) and an own build machine.
 
 ## Description of scripts 
 
+- `../.github/workflows` contains jobs run by GitHub Actions.
+
 - `../.circleci/config.yml` describing the build jobs that are run
-  by Circle-CI
+  by CircleCI.
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -26,8 +30,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    ci_scripts/manual_remote_tests.sh rust
-    ci_scripts/manual_remote_tests.sh python
+    scripts/manual_remote_tests.sh rust
+    scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -41,6 +45,10 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd ci_scripts  # where all CI things are 
+    cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
+
+Additionally, you can install qemu and build arm64 docker image:
+    apt-get install qemu binfmt-support qemu-user-static
+    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

scripts/ci_upload.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
+
+
+# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
+export BRANCH=${CIRCLE_BRANCH:master}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
+
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
+
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

scripts/coverage.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+set -eu
+
+if ! which grcov 2>/dev/null 1>&2; then
+    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
+    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
+    exit 1
+fi
+
+# Allow `-Z` flags without using nightly Rust.
+export RUSTC_BOOTSTRAP=1
+
+# We are using `-Zprofile` instead of source-based coverage [1]
+# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
+#
+# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
+# [2] https://github.com/mozilla/grcov/issues/595
+
+export CARGO_INCREMENTAL=0
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTDOCFLAGS="-Cpanic=abort"
+cargo clean
+cargo build
+cargo test
+
+grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2010_x86_64
+FROM quay.io/pypa/manylinux2014_aarch64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate 
+# Install a recent Perl, needed to install the openssl crate
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.6 environment as the base environment 
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+
+popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+#
+# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
+#
+# Avoid using rustup here as it depends on reading /proc/self/exe and
+# has problems running under QEMU.
+curl "https://static.rust-lang.org/dist/rust-1.52.1-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
+cd "rust-1.52.1-$(uname -m)-unknown-linux-gnu"
+./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+rustc --version
+cd ..
+rm -fr "rust-1.52.1-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -0,0 +1,21 @@
+FROM quay.io/pypa/manylinux2014_x86_64
+
+# Configure ld.so/ldconfig and pkg-config
+RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
+    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
+ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
+
+# Install a recent Perl, needed to install the openssl crate
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_openssl.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e -x
+
+OPENSSL_VERSION=1.1.1a
+OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
+
+curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
+echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
+tar xzf openssl-${OPENSSL_VERSION}.tar.gz
+cd openssl-${OPENSSL_VERSION} 
+./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
+
+sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
+sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
+sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
+
+make depend
+make 
+make install_sw install_ssldirs
+ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_perl.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
+
+./Configure -de
+make
+make install

scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.6 environment as the base environment
+/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.6.*/bin/tox
+ln -s /opt/_internal/cpython-3.6.*/bin/devpi
+ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
+
+popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "1.50.0-$(uname -m)-unknown-linux-gnu" -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf "/root/.rustup/toolchains/1.50.0-$(uname -m)-unknown-linux-gnu/share"

scripts/manual_remote_tests.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -xe
+
+JOB=${1:?need to specify 'rust' or 'python'}
+BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
+REPONAME="$(basename $(git rev-parse --show-toplevel))"
+
+time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"

scripts/remote_python_packaging.sh

@@ -1,7 +1,7 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
@@ -15,6 +15,7 @@ git ls-files >.rsynclist
 # we seem to need .git for setuptools_scm versioning 
 find .git >>.rsynclist
 rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
 
 set +x
 
@@ -26,25 +27,41 @@ set +x
 # everything is terminated/cleaned up and we have no orphaned
 # useless still-running docker-containers consuming resources. 
 
-ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+for arch in "" "-arm64"; do
+
+ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
     set +x -e
     shopt -s huponexit
-    cd $BUILDDIR
-    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    cd ${BUILDDIR}${arch}
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
     set -x
 
     # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+    docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+       deltachat/coredeps${arch} scripts/run_all.sh
 
 _HERE
 
+done
+
 echo "--- Running $CIRCLE_JOB remotely"
 
-ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+echo "--- Building aarch64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
+
+echo "--- Building x86_64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+
 mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+
+# Wheels
+for arch in "" "-arm64"; do
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+done
+
+# Source packages
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
+
+# Documentation
 rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

scripts/remote_tests_python.sh

@@ -1,10 +1,9 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILD_ID=${1:?specify build ID}
 
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -18,7 +17,7 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running $CIRCLE_JOB remotely"
+echo "--- Running Python tests remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
@@ -29,7 +28,6 @@ ssh $SSHTARGET <<_HERE
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     export TARGET=release
-    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -43,5 +41,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash ci_scripts/run-python-test.sh 
+    bash scripts/run-python-test.sh 
 _HERE

scripts/remote_tests_rust.sh

@@ -0,0 +1,29 @@
+#!/bin/bash 
+
+BUILD_ID=${1:?specify build ID}
+
+SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+BUILDDIR=ci_builds/$BUILD_ID
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running Rust tests remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
+    cd $BUILDDIR
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash scripts/run-rust-test.sh
+_HERE
+

scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

scripts/run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   ./run-integration-tests.sh -e py35
+#   scripts/run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   ./run-integration-tests.sh -e py35 -- -x
+#   scripts/run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,9 +23,6 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
-if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
-    export DCC_PY_LIVECONFIG=liveconfig
-fi
 tox "$@"
 ret=$?
 popd

scripts/run_all.sh

@@ -19,14 +19,17 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PATH=$PATH:/opt/python/cp36-cp36m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.5
-ln -s /opt/python/cp35-cp35m/bin/python3.5
+rm -f python3.6
 ln -s /opt/python/cp36-cp36m/bin/python3.6
+rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
+rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
+rm -f python3.9
+ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -39,9 +42,8 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
-unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
 popd
 
 

scripts/set_core_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys
@@ -63,13 +63,14 @@ def main():
     ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
-    for line in open("CHANGELOG.md"):
-        ## 1.25.0
-        if line.startswith("## "):
-            if line[2:].strip().startswith(newversion):
-                break
-    else:
-        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+    if "alpha" not in newversion:
+        for line in open("CHANGELOG.md"):
+            ## 1.25.0
+            if line.startswith("## "):
+                if line[2:].strip().startswith(newversion):
+                    break
+        else:
+            raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
@@ -81,9 +82,10 @@ def main():
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit make sure to: ")
+    print("after commit, on master make sure to: ")
     print("")
-    print("   git tag {}".format(newversion))
+    print("   git tag -a {}".format(newversion))
+    print("   git push origin {}".format(newversion))
     print("")
 
 

src/accounts.rs

@@ -6,11 +6,10 @@ use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
-use anyhow::{ensure, Context as _};
+use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::error::Result;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -136,15 +135,25 @@ impl Accounts {
         let old_id = self.config.get_selected_account().await;
 
         // create new account
-        let account_config = self.config.new_account(&self.dir).await?;
+        let account_config = self
+            .config
+            .new_account(&self.dir)
+            .await
+            .context("failed to create new account")?;
 
         let new_dbfile = account_config.dbfile().into();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
 
         let res = {
-            fs::create_dir_all(&account_config.dir).await?;
-            fs::rename(&dbfile, &new_dbfile).await?;
-            fs::rename(&blobdir, &new_blobdir).await?;
+            fs::create_dir_all(&account_config.dir)
+                .await
+                .context("failed to create dir")?;
+            fs::rename(&dbfile, &new_dbfile)
+                .await
+                .context("failed to rename dbfile")?;
+            fs::rename(&blobdir, &new_blobdir)
+                .await
+                .context("failed to rename blobdir")?;
             Ok(())
         };
 
@@ -188,7 +197,7 @@ impl Accounts {
         let id = self.add_account().await?;
         let ctx = self.get_account(id).await.expect("just added");
 
-        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
             Ok(_) => Ok(id),
             Err(err) => {
                 // remove temp account
@@ -347,7 +356,6 @@ impl Config {
 
             inner.accounts.push(AccountConfig {
                 id,
-                name: String::new(),
                 dir: target_dir.into(),
                 uuid,
             });
@@ -414,8 +422,6 @@ impl Config {
 pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
-    /// Display name
-    pub name: String,
     /// Root directory for all data for this account.
     pub dir: std::path::PathBuf,
     pub uuid: Uuid,
@@ -506,7 +512,10 @@ mod tests {
         let ctx = accounts.get_selected_account().await;
         assert_eq!(
             "me@mail.com",
-            ctx.get_config(crate::config::Config::Addr).await.unwrap()
+            ctx.get_config(crate::config::Config::Addr)
+                .await
+                .unwrap()
+                .unwrap()
         );
     }
 

src/aheader.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::*;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -1,5 +1,6 @@
 //! # Blob directory management
 
+use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 
@@ -7,14 +8,21 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::format_err;
+use anyhow::Context as _;
+use anyhow::Error;
+use image::DynamicImage;
 use image::GenericImageView;
+use image::ImageFormat;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::*;
+use crate::constants::{
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
+};
 use crate::context::Context;
-use crate::error::Error;
 use crate::events::EventType;
 use crate::message;
 
@@ -50,12 +58,12 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: impl AsRef<str>,
+        suggested_name: &str,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -63,6 +71,12 @@ impl<'a> BlobObject<'a> {
                 blobname: name.clone(),
                 cause: err.into(),
             })?;
+
+        // workaround a bug in async-std
+        // (the executor does not handle blocking operation in Drop correctly,
+        // see https://github.com/async-rs/async-std/issues/900 )
+        let _ = file.flush().await;
+
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
@@ -123,18 +137,17 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: impl AsRef<Path>,
+        src: &Path,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file =
-            fs::File::open(src.as_ref())
-                .await
-                .map_err(|err| BlobError::CopyFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: String::from(""),
-                    src: src.as_ref().to_path_buf(),
-                    cause: err,
-                })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let mut src_file = fs::File::open(src)
+            .await
+            .map_err(|err| BlobError::CopyFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: String::from(""),
+                src: src.to_path_buf(),
+                cause: err,
+            })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
@@ -147,10 +160,14 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.as_ref().to_path_buf(),
+                src: src.to_path_buf(),
                 cause: err,
             });
         }
+
+        // workaround, see create() for details
+        let _ = dst_file.flush().await;
+
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
@@ -177,16 +194,13 @@ impl<'a> BlobObject<'a> {
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &Context,
-        src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject<'_>, BlobError> {
-        if src.as_ref().starts_with(context.get_blobdir()) {
+        context: &'a Context,
+        src: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        if src.starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.as_ref().starts_with("$BLOBDIR/") {
-            BlobObject::from_name(
-                context,
-                src.as_ref().to_str().unwrap_or_default().to_string(),
-            )
+        } else if src.starts_with("$BLOBDIR/") {
+            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -207,23 +221,22 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &Context,
-        path: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject, BlobError> {
-        let rel_path = path
-            .as_ref()
-            .strip_prefix(context.get_blobdir())
-            .map_err(|_| BlobError::WrongBlobdir {
-                blobdir: context.get_blobdir().to_path_buf(),
-                src: path.as_ref().to_path_buf(),
-            })?;
-        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+        context: &'a Context,
+        path: &Path,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let rel_path =
+            path.strip_prefix(context.get_blobdir())
+                .map_err(|_| BlobError::WrongBlobdir {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    src: path.to_path_buf(),
+                })?;
+        if !BlobObject::is_acceptible_blob_name(rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.as_ref().to_path_buf(),
+                blobname: path.to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.as_ref().to_path_buf(),
+            blobname: path.to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -367,26 +380,25 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
-        })?;
 
-        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
-            return Ok(());
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
+                MediaQuality::Worse => WORSE_AVATAR_SIZE,
+            };
+
+        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
+        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
+        if let Some(new_name) = self
+            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
+            .await?
+        {
+            self.name = new_name;
         }
-
-        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
         Ok(())
     }
 
@@ -398,40 +410,140 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img_wh =
+            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
+                .unwrap_or_default()
+            {
+                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
+                MediaQuality::Worse => WORSE_IMAGE_SIZE,
+            };
+
+        if self
+            .recode_to_size(context, blob_abs, img_wh, None)
+            .await?
+            .is_some()
+        {
+            return Err(format_err!(
+                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
+            )
+            .into());
+        }
+        Ok(())
+    }
+
+    async fn recode_to_size(
+        &self,
+        context: &Context,
+        mut blob_abs: PathBuf,
+        mut img_wh: u32,
+        max_bytes: Option<usize>,
+    ) -> Result<Option<String>, BlobError> {
+        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
+        let orientation = self.get_exif_orientation(context);
+        let mut encoded = Vec::new();
+        let mut changed_name = None;
 
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
+        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
+            encoded.clear();
+            img.write_to(encoded, image::ImageFormat::Jpeg)?;
+            Ok(())
+        }
+        fn encode_img_exceeds_bytes(
+            context: &Context,
+            img: &DynamicImage,
+            max_bytes: Option<usize>,
+            encoded: &mut Vec<u8>,
+        ) -> anyhow::Result<bool> {
+            if let Some(max_bytes) = max_bytes {
+                encode_img(img, encoded)?;
+                if encoded.len() > max_bytes {
+                    info!(
+                        context,
+                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
+                        encoded.len(),
+                        img.width(),
+                        img.height(),
+                        max_bytes,
+                    );
+                    return Ok(true);
+                }
+            }
+            Ok(false)
+        }
+        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
 
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
+        let do_scale =
+            exceeds_width || encode_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
+        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
+
+        if do_scale || do_rotate {
+            if do_rotate {
+                img = match orientation {
+                    Ok(90) => img.rotate90(),
+                    Ok(180) => img.rotate180(),
+                    Ok(270) => img.rotate270(),
+                    _ => img,
+                }
+            }
+
+            if do_scale {
+                if !exceeds_width {
+                    // The image is already smaller than img_wh, but exceeds max_bytes
+                    // We can directly start with trying to scale down to 2/3 of its current width
+                    img_wh = max(img.width(), img.height()) * 2 / 3
+                }
+
+                loop {
+                    let new_img = img.thumbnail(img_wh, img_wh);
+
+                    if encode_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
+                        if img_wh < 20 {
+                            return Err(format_err!(
+                                "Failed to scale image to below {}B",
+                                max_bytes.unwrap_or_default()
+                            )
+                            .into());
+                        }
+
+                        img_wh = img_wh * 2 / 3;
+                    } else {
+                        info!(
+                            context,
+                            "Final scaled-down image size: {}B ({}px)",
+                            encoded.len(),
+                            img_wh
+                        );
+                        break;
+                    }
+                }
+            }
+
+            // The file format is JPEG now, we may have to change the file extension
+            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
+                blob_abs = blob_abs.with_extension("jpg");
+                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
+                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
+                changed_name = Some(format!("$BLOBDIR/{}", file_name));
+            }
+
+            if encoded.is_empty() {
+                encode_img(&img, &mut encoded)?;
+            }
+
+            fs::write(&blob_abs, &encoded)
+                .await
+                .map_err(|err| BlobError::WriteFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                    cause: err.into(),
+                })?;
         }
 
-        let mut img = img.thumbnail(img_wh, img_wh);
-        match self.get_exif_orientation(context) {
-            Ok(90) => img = img.rotate90(),
-            Ok(180) => img = img.rotate180(),
-            Ok(270) => img = img.rotate270(),
-            _ => {}
-        }
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
+        Ok(changed_name)
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -495,75 +607,67 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
+    #[error("{0}")]
+    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
+    use fs::File;
+
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
-        let fname = t.ctx.get_blobdir().join("foo");
+        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
+        let fname = t.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let foo_path = t.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.txt", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -579,15 +683,15 @@ mod tests {
     #[async_std::test]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
+        BlobObject::create(&t, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
+        BlobObject::create(&t, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -605,7 +709,7 @@ mod tests {
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
@@ -615,14 +719,16 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
-        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
+            .await
+            .is_err());
+        let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
@@ -632,14 +738,16 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.ctx.get_blobdir().join("internal");
+        let src_int = t.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -649,7 +757,9 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+            .await
+            .unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -704,4 +814,105 @@ mod tests {
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
     }
+
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(&avatar_blob).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+
+        async fn file_size(path_buf: &PathBuf) -> u64 {
+            let file = File::open(path_buf).await.unwrap();
+            file.metadata().await.unwrap().len()
+        }
+
+        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
+
+        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
+            .await
+            .unwrap();
+        assert!(file_size(&avatar_blob).await <= 3000);
+        assert!(file_size(&avatar_blob).await > 2000);
+        let img = image::open(&avatar_blob).unwrap();
+        assert!(img.width() > 130);
+        assert_eq!(img.width(), img.height());
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
+        assert_eq!(
+            avatar_cfg,
+            avatar_src.with_extension("jpg").to_str().unwrap()
+        );
+
+        let img = image::open(avatar_cfg).unwrap();
+        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
+        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        let avatar_blob = t.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
 }

src/chatlist.rs

@@ -1,15 +1,19 @@
 //! # Chat list module
 
-use crate::chat;
-use crate::chat::*;
-use crate::constants::*;
-use crate::contact::*;
-use crate::context::*;
+use anyhow::{bail, ensure, Result};
+
+use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
+use crate::constants::{
+    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
+    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+};
+use crate::contact::Contact;
+use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
-use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -58,9 +62,8 @@ impl Chatlist {
     ///   messages from addresses that have no relationship to the configured account.
     ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
     ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
-    ///   or "Not now".
-    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
+    ///   and offers the options "Start chat", "Block" and "Not now";
+    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -118,10 +121,9 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .await
+            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+                .await?
                 .unwrap_or_default()
-                .0
         } else {
             ChatId::new(0)
         };
@@ -228,10 +230,9 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .await
+                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+                    .await?
                     .unwrap_or_default()
-                    .0
             } else {
                 ChatId::new(0)
             };
@@ -256,13 +257,11 @@ impl Chatlist {
                 process_rows,
             ).await?;
             if !flag_no_specials {
-                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
+                if let Some(last_deaddrop_fresh_msg_id) =
+                    get_last_deaddrop_fresh_msg(context).await?
                 {
                     if !flag_for_forwarding {
-                        ids.insert(
-                            0,
-                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
-                        );
+                        ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                     }
                 }
                 add_archived_link_item = true;
@@ -270,11 +269,11 @@ impl Chatlist {
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -359,28 +358,29 @@ impl Chatlist {
             return ret;
         };
 
-        let mut lastcontact = None;
-
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
-            }
-
-            Some(lastmsg)
-        } else {
-            None
-        };
+        let (lastmsg, lastcontact) =
+            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+                if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                    (Some(lastmsg), None)
+                } else {
+                    match chat.typ {
+                        Chattype::Group | Chattype::Mailinglist => {
+                            let lastcontact =
+                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                            (Some(lastmsg), lastcontact)
+                        }
+                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
+                    }
+                }
+            } else {
+                (None, None)
+            };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(
-                context
-                    .stock_str(StockMessage::NoMessages)
-                    .await
-                    .to_string(),
-            );
+            ret.text2 = Some(stock_str::no_messages(context).await);
         } else {
             ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
                 .await;
@@ -395,25 +395,23 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
-    context
+pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
+    let count = context
         .sql
-        .query_get_value(
-            context,
+        .count(
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
             paramsv![],
         )
-        .await
-        .unwrap_or_default()
+        .await?;
+    Ok(count)
 }
 
-async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>> {
     // We have an index over the state-column, this should be
     // sufficient as there are typically only few fresh messages.
-    context
+    let id = context
         .sql
         .query_get_value(
-            context,
             concat!(
                 "SELECT m.id",
                 " FROM msgs m",
@@ -426,30 +424,37 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
             ),
             paramsv![],
         )
-        .await
+        .await?;
+    Ok(id)
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
+    use crate::constants::Viewtype;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message;
+    use crate::message::ContactRequestDecision;
+    use crate::stock_str::StockMessage;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
+        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
+        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -458,26 +463,24 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .set_visibility(&t, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -486,23 +489,23 @@ mod tests {
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        t.update_device_chats().await.unwrap();
+        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -511,49 +514,177 @@ mod tests {
     #[async_std::test]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
+        t.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
     }
 
+    #[async_std::test]
+    async fn test_search_single_chat() -> anyhow::Result<()> {
+        let t = TestContext::new_alice().await;
+
+        // receive a one-to-one-message, accept contact request
+        dc_receive_imf(
+            &t,
+            b"From: Bob Authname <bob@example.org>\n\
+                 To: alice@example.com\n\
+                 Subject: foo\n\
+                 Message-ID: <msg1234@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
+                 \n\
+                 hello foo\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+
+        let chats = Chatlist::try_load(&t, 0, Some("Bob Authname"), None).await?;
+        assert_eq!(chats.len(), 0);
+
+        let msg = t.get_last_msg().await;
+        assert_eq!(msg.get_chat_id(), DC_CHAT_ID_DEADDROP);
+
+        let chat_id =
+            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
+                .await
+                .unwrap();
+        let contacts = get_chat_contacts(&t, chat_id).await?;
+        let contact_id = *contacts.first().unwrap();
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "Bob Authname");
+
+        // check, the one-to-one-chat can be found using chatlist search query
+        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
+        assert_eq!(chats.len(), 1);
+        assert_eq!(chats.get_chat_id(0), chat_id);
+
+        // change the name of the contact; this also changes the name of the one-to-one-chat
+        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
+        assert_eq!(contact_id, test_id);
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "Bob Nickname");
+        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
+        assert_eq!(chats.len(), 0);
+        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
+        assert_eq!(chats.len(), 1);
+
+        // revert contact to authname, this again changes the name of the one-to-one-chat
+        let test_id = Contact::create(&t, "", "bob@example.org").await?;
+        assert_eq!(contact_id, test_id);
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "Bob Authname");
+        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
+        assert_eq!(chats.len(), 1);
+        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
+        assert_eq!(chats.len(), 0);
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
+        let t = TestContext::new_alice().await;
+
+        // receive a one-to-one-message without authname set, accept contact request
+        dc_receive_imf(
+            &t,
+            b"From: bob@example.org\n\
+                 To: alice@example.com\n\
+                 Subject: foo\n\
+                 Message-ID: <msg5678@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
+                 \n\
+                 hello foo\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await?;
+
+        let msg = t.get_last_msg().await;
+        let chat_id =
+            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
+                .await
+                .unwrap();
+        let contacts = get_chat_contacts(&t, chat_id).await?;
+        let contact_id = *contacts.first().unwrap();
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "bob@example.org");
+
+        // check, the one-to-one-chat can be found using chatlist search query
+        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
+        assert_eq!(chats.len(), 1);
+        assert_eq!(chats.get_chat_id(0), chat_id);
+
+        // change the name of the contact; this also changes the name of the one-to-one-chat
+        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
+        assert_eq!(contact_id, test_id);
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "Bob Nickname");
+        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
+        assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
+        let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
+        assert_eq!(chats.len(), 1);
+        assert_eq!(chats.get_chat_id(0), chat_id);
+
+        // revert name change, this again changes the name of the one-to-one-chat to the email-address
+        let test_id = Contact::create(&t, "", "bob@example.org").await?;
+        assert_eq!(contact_id, test_id);
+        let chat = Chat::load_from_db(&t, chat_id).await?;
+        assert_eq!(chat.get_name(), "bob@example.org");
+        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
+        assert_eq!(chats.len(), 1);
+        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
+        assert_eq!(chats.len(), 0);
+
+        // finally, also check that a simple substring-search is working with email-addresses
+        let chats = Chatlist::try_load(&t, 0, Some("b@exa"), None).await?;
+        assert_eq!(chats.len(), 1);
+        let chats = Chatlist::try_load(&t, 0, Some("b@exac"), None).await?;
+        assert_eq!(chats.len(), 0);
+
+        Ok(())
+    }
+
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
+        chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -0,0 +1,62 @@
+//! Implementation of Consistent Color Generation
+//!
+//! Consistent Color Generation is defined in XEP-0392.
+//!
+//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
+//! corresponding settings.
+use hsluv::hsluv_to_rgb;
+use sha1::{Digest, Sha1};
+
+/// Converts an identifier to Hue angle.
+fn str_to_angle(s: &str) -> f64 {
+    let bytes = s.as_bytes();
+    let result = Sha1::digest(bytes);
+    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
+        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
+    f64::from(checksum) / 65536.0 * 360.0
+}
+
+/// Converts RGB tuple to a 24-bit number.
+///
+/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
+/// most significant bits corresponding to the red color.
+fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
+    let r = ((r * 256.0) as u32).min(255);
+    let g = ((g * 256.0) as u32).min(255);
+    let b = ((b * 256.0) as u32).min(255);
+    65536 * r + 256 * g + b
+}
+
+/// Converts an identifier to RGB color.
+///
+/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
+/// half (50.0) to make colors suitable both for light and dark theme.
+pub(crate) fn str_to_color(s: &str) -> u32 {
+    rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_str_to_angle() {
+        // Test against test vectors from
+        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
+        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
+        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
+        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
+        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
+        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
+    }
+
+    #[test]
+    fn test_rgb_to_u32() {
+        assert_eq!(rgb_to_u32((0.0, 0.0, 0.0)), 0);
+        assert_eq!(rgb_to_u32((1.0, 1.0, 1.0)), 0xffffff);
+        assert_eq!(rgb_to_u32((0.0, 0.0, 1.0)), 0x0000ff);
+        assert_eq!(rgb_to_u32((0.0, 1.0, 0.0)), 0x00ff00);
+        assert_eq!(rgb_to_u32((1.0, 0.0, 0.0)), 0xff0000);
+        assert_eq!(rgb_to_u32((1.0, 0.5, 0.0)), 0xff8000);
+    }
+}

src/config.rs

@@ -1,5 +1,6 @@
 //! # Key-value configuration management
 
+use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -7,12 +8,13 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
 use crate::events::EventType;
 use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::stock::StockMessage;
+use crate::provider::{get_provider_by_id, Provider};
+use crate::stock_str;
 
 /// The available configuration keys.
 #[derive(
@@ -35,9 +37,6 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    #[strum(props(default = "INBOX"))]
-    ImapFolder,
-
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -63,14 +62,20 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
+    #[strum(props(default = "0"))]
+    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
+
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    /// If set to "1", on the first time `start_io()` is called after configuring,
+    /// the newest existing messages are fetched.
+    /// Existing recipients are added to the contact database regardless of this setting.
     #[strum(props(default = "1"))]
-    FetchExisting,
+    FetchExistingMsgs,
 
     #[strum(props(default = "0"))]
     KeyGenType,
@@ -113,6 +118,9 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
+    ConfiguredSpamFolder,
+    ConfiguredTimestamp,
+    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -133,106 +141,142 @@ pub enum Config {
 
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
+
+    /// Timestamp of the last time housekeeping was run
+    LastHousekeeping,
+
+    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
+    #[strum(props(default = "60"))]
+    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
-    pub async fn config_exists(&self, key: Config) -> bool {
-        self.sql.get_raw_config(self, key).await.is_some()
+    pub async fn config_exists(&self, key: Config) -> Result<bool> {
+        Ok(self.sql.get_raw_config(key).await?.is_some())
     }
 
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub async fn get_config(&self, key: Config) -> Option<String> {
+    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(self, key).await;
+                let rel_path = self.sql.get_raw_config(key).await?;
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(self, key).await,
+            _ => self.sql.get_raw_config(key).await?,
         };
 
         if value.is_some() {
-            return value;
+            return Ok(value);
         }
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
-            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
-            _ => key.get_str("default").map(|s| s.to_string()),
+            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
+            Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
+            _ => Ok(key.get_str("default").map(|s| s.to_string())),
         }
     }
 
-    pub async fn get_config_int(&self, key: Config) -> i32 {
+    pub async fn get_config_int(&self, key: Config) -> Result<i32> {
         self.get_config(key)
             .await
-            .and_then(|s| s.parse().ok())
-            .unwrap_or_default()
+            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
     }
 
-    pub async fn get_config_bool(&self, key: Config) -> bool {
-        self.get_config_int(key).await != 0
+    pub async fn get_config_i64(&self, key: Config) -> Result<i64> {
+        self.get_config(key)
+            .await
+            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
+    }
+
+    pub async fn get_config_u64(&self, key: Config) -> Result<u64> {
+        self.get_config(key)
+            .await
+            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
+    }
+
+    pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
+        Ok(self.get_config_int(key).await? != 0)
     }
 
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
-    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter).await {
-            0 => None,
-            1 => Some(0),
-            x => Some(x as i64),
+    pub async fn get_config_delete_server_after(&self) -> Result<Option<i64>> {
+        match self.get_config_int(Config::DeleteServerAfter).await? {
+            0 => Ok(None),
+            1 => Ok(Some(0)),
+            x => Ok(Some(x as i64)),
         }
     }
 
+    /// Gets the configured provider, as saved in the `configured_provider` value.
+    ///
+    /// The provider is determined by `get_provider_info()` during configuration and then saved
+    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
+    pub async fn get_configured_provider(&self) -> Result<Option<&'static Provider>> {
+        if let Some(cfg) = self.get_config(Config::ConfiguredProvider).await? {
+            return Ok(get_provider_by_id(&cfg));
+        }
+        Ok(None)
+    }
+
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
     /// after `x` seconds.
-    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter).await {
-            0 => None,
-            x => Some(x as i64),
+    pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
+        match self.get_config_int(Config::DeleteDeviceAfter).await? {
+            0 => Ok(None),
+            x => Ok(Some(x as i64)),
         }
     }
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
                     .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![])
                     .await?;
                 self.sql
-                    .set_raw_config_bool(self, "attach_selfavatar", true)
+                    .set_raw_config_bool("attach_selfavatar", true)
                     .await?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value).await?;
-                        blob.recode_to_avatar_size(self)?;
-                        self.sql
-                            .set_raw_config(self, key, Some(blob.as_name()))
-                            .await
+                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
+                        blob.recode_to_avatar_size(self).await?;
+                        self.sql.set_raw_config(key, Some(blob.as_name())).await?;
+                        Ok(())
+                    }
+                    None => {
+                        self.sql.set_raw_config(key, None).await?;
+                        Ok(())
                     }
-                    None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine).await;
+                let def = stock_str::status_line(self).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(self, key, val).await
+                self.sql.set_raw_config(key, val).await?;
+                Ok(())
             }
             Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
+                let ret = self
+                    .sql
+                    .set_raw_config(key, value)
+                    .await
+                    .map_err(Into::into);
                 // Force chatlist reload to delete old messages immediately.
                 self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
@@ -242,16 +286,30 @@ impl Context {
             }
             Config::Displayname => {
                 let value = value.map(improve_single_line_input);
-                self.sql.set_raw_config(self, key, value.as_deref()).await
+                self.sql.set_raw_config(key, value.as_deref()).await?;
+                Ok(())
             }
             Config::DeleteServerAfter => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
+                let ret = self
+                    .sql
+                    .set_raw_config(key, value)
+                    .await
+                    .map_err(Into::into);
                 job::schedule_resync(self).await;
                 ret
             }
-            _ => self.sql.set_raw_config(self, key, value).await,
+            _ => {
+                self.sql.set_raw_config(key, value).await?;
+                Ok(())
+            }
         }
     }
+
+    pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
+        self.set_config(key, if value { Some("1") } else { None })
+            .await?;
+        Ok(())
+    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -273,12 +331,8 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::constants::AVATAR_SIZE;
-    use crate::test_utils::*;
-    use image::GenericImageView;
+    use crate::test_utils::TestContext;
     use num_traits::FromPrimitive;
-    use std::fs::File;
-    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -292,104 +346,17 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_default_prop() {
-        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
-        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.ctx
-            .set_config(Config::MediaQuality, Some("1"))
-            .await
-            .unwrap();
+        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
 
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
+        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure.rs

@@ -12,17 +12,20 @@ use itertools::Itertools;
 use job::Action;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
-use crate::config::Config;
-use crate::dc_tools::*;
+use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
-use crate::oauth2::*;
+use crate::oauth2::dc_get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 use crate::{chat, e2ee, provider};
-use crate::{constants::*, job};
+use crate::{config::Config, dc_tools::time};
+use crate::{
+    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
+    job,
+};
 use crate::{context::Context, param::Params};
 
 use auto_mozilla::moz_autoconfigure;
@@ -47,8 +50,11 @@ macro_rules! progress {
 
 impl Context {
     /// Checks if the context is already configured.
-    pub async fn is_configured(&self) -> bool {
-        self.sql.get_raw_config_bool(self, "configured").await
+    pub async fn is_configured(&self) -> Result<bool> {
+        self.sql
+            .get_raw_config_bool("configured")
+            .await
+            .map_err(Into::into)
     }
 
     /// Configures this account with the currently set parameters.
@@ -81,14 +87,14 @@ impl Context {
     async fn inner_configure(&self) -> Result<()> {
         info!(self, "Configure ...");
 
-        let mut param = LoginParam::from_database(self, "").await;
+        let mut param = LoginParam::from_database(self, "").await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = provider::get_provider_info(&param.addr) {
+        if let Some(provider) = param.provider {
             if let Some(config_defaults) = &provider.config_defaults {
                 for def in config_defaults.iter() {
-                    if !self.config_exists(def.key).await {
+                    if !self.config_exists(def.key).await? {
                         info!(self, "apply config_defaults {}={}", def.key, def.value);
                         self.set_config(def.key, Some(def.value)).await?;
                     } else {
@@ -124,9 +130,10 @@ impl Context {
                     self,
                     0,
                     Some(
-                        self.stock_string_repl_str(
-                            StockMessage::ConfigurationFailed,
-                            // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        stock_str::configuration_failed(
+                            self,
+                            // We are using Anyhow's .context() and to show the
+                            // inner error, too, we need the {:#}:
                             format!("{:#}", err),
                         )
                         .await
@@ -163,8 +170,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
-    //let ctx2 = ctx.clone();
-    //let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
+    let ctx2 = ctx.clone();
+    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
@@ -173,13 +180,13 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
         if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
-            .await
+            .await?
             .and_then(|e| e.parse().ok())
         {
             info!(ctx, "Authorized address is {}", oauth2_addr);
             param.addr = oauth2_addr;
             ctx.sql
-                .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
+                .set_raw_config("addr", Some(param.addr.as_str()))
                 .await?;
         }
         progress!(ctx, 20);
@@ -205,9 +212,51 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     {
         // no advanced parameters entered by the user: query provider-database or do Autoconfig
 
-        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
-            param_autoconfig = Some(servers);
+        info!(
+            ctx,
+            "checking internal provider-info for offline autoconfig"
+        );
+
+        if let Some(provider) = provider::get_provider_info(&param_domain).await {
+            param.provider = Some(provider);
+            match provider.status {
+                provider::Status::Ok | provider::Status::Preparation => {
+                    if provider.server.is_empty() {
+                        info!(ctx, "offline autoconfig found, but no servers defined");
+                        param_autoconfig = None;
+                    } else {
+                        info!(ctx, "offline autoconfig found");
+                        let servers = provider
+                            .server
+                            .iter()
+                            .map(|s| ServerParams {
+                                protocol: s.protocol,
+                                socket: s.socket,
+                                hostname: s.hostname.to_string(),
+                                port: s.port,
+                                username: match s.username_pattern {
+                                    UsernamePattern::Email => param.addr.to_string(),
+                                    UsernamePattern::Emaillocalpart => {
+                                        if let Some(at) = param.addr.find('@') {
+                                            param.addr.split_at(at).0.to_string()
+                                        } else {
+                                            param.addr.to_string()
+                                        }
+                                    }
+                                },
+                            })
+                            .collect();
+
+                        param_autoconfig = Some(servers)
+                    }
+                }
+                provider::Status::Broken => {
+                    info!(ctx, "offline autoconfig found, provider is broken");
+                    param_autoconfig = None;
+                }
+            }
         } else {
+            info!(ctx, "no offline autoconfig found");
             param_autoconfig =
                 get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
@@ -220,10 +269,10 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let mut servers = param_autoconfig.unwrap_or_default();
     if !servers
         .iter()
-        .any(|server| server.protocol == Protocol::IMAP)
+        .any(|server| server.protocol == Protocol::Imap)
     {
         servers.push(ServerParams {
-            protocol: Protocol::IMAP,
+            protocol: Protocol::Imap,
             hostname: param.imap.server.clone(),
             port: param.imap.port,
             socket: param.imap.security,
@@ -232,10 +281,10 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     }
     if !servers
         .iter()
-        .any(|server| server.protocol == Protocol::SMTP)
+        .any(|server| server.protocol == Protocol::Smtp)
     {
         servers.push(ServerParams {
-            protocol: Protocol::SMTP,
+            protocol: Protocol::Smtp,
             hostname: param.smtp.server.clone(),
             port: param.smtp.port,
             socket: param.smtp.security,
@@ -254,9 +303,10 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     let smtp_addr = param.addr.clone();
     let smtp_servers: Vec<ServerParams> = servers
         .iter()
-        .filter(|params| params.protocol == Protocol::SMTP)
+        .filter(|params| params.protocol == Protocol::Smtp)
         .cloned()
         .collect();
+    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -267,8 +317,15 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             smtp_param.port = smtp_server.port;
             smtp_param.security = smtp_server.socket;
 
-            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
-                .await
+            match try_smtp_one_param(
+                &context_smtp,
+                &smtp_param,
+                &smtp_addr,
+                oauth2,
+                provider_strict_tls,
+                &mut smtp,
+            )
+            .await
             {
                 Ok(_) => {
                     smtp_configured = true;
@@ -288,13 +345,13 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 600);
 
     // Configure IMAP
-    let (_s, r) = async_std::sync::channel(1);
+    let (_s, r) = async_std::channel::bounded(1);
     let mut imap = Imap::new(r);
 
     let mut imap_configured = false;
     let imap_servers: Vec<&ServerParams> = servers
         .iter()
-        .filter(|params| params.protocol == Protocol::IMAP)
+        .filter(|params| params.protocol == Protocol::Imap)
         .collect();
     let imap_servers_count = imap_servers.len();
     let mut errors = Vec::new();
@@ -304,7 +361,16 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+        match try_imap_one_param(
+            ctx,
+            &param.imap,
+            &param.addr,
+            oauth2,
+            provider_strict_tls,
+            &mut imap,
+        )
+        .await
+        {
             Ok(_) => {
                 imap_configured = true;
                 break;
@@ -334,8 +400,8 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     progress!(ctx, 900);
 
-    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
-        || ctx.get_config_bool(Config::MvboxMove).await;
+    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
+        || ctx.get_config_bool(Config::MvboxMove).await?;
 
     imap.configure_folders(ctx, create_mvbox).await?;
 
@@ -350,7 +416,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     // "configured_" prefix; also write the "configured"-flag */
     // the trailing underscore is correct
     param.save_to_database(ctx, "configured_").await?;
-    ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+    ctx.sql.set_raw_config_bool("configured", true).await?;
+    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+        .await?;
 
     progress!(ctx, 920);
 
@@ -364,71 +432,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     .await;
 
     progress!(ctx, 940);
-    //update_device_chats_handle.await?;
+    update_device_chats_handle.await?;
 
     Ok(())
 }
 
-#[derive(Debug, PartialEq, Eq)]
-enum AutoconfigProvider {
-    Mozilla,
-    Outlook,
-}
-
-#[derive(Debug, PartialEq, Eq)]
-struct AutoconfigSource {
-    provider: AutoconfigProvider,
-    url: String,
-}
-
-impl AutoconfigSource {
-    fn all(domain: &str, addr: &str) -> [Self; 5] {
-        [
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr,
-                ),
-            },
-            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!(
-                    "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-                    domain, addr
-                ),
-            },
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
-            },
-            // Outlook uses always SSL but different domains (this comment describes the next two steps)
-            AutoconfigSource {
-                provider: AutoconfigProvider::Outlook,
-                url: format!(
-                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
-                    domain
-                ),
-            },
-            // always SSL for Thunderbird's database
-            AutoconfigSource {
-                provider: AutoconfigProvider::Mozilla,
-                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
-            },
-        ]
-    }
-
-    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
-        let params = match self.provider {
-            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
-            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
-        };
-
-        Ok(params)
-    }
-}
-
 /// Retrieve available autoconfigurations.
 ///
 /// A Search configurations from the domain used in the email-address, prefer encrypted
@@ -439,74 +447,79 @@ async fn get_autoconfig(
     param_domain: &str,
     param_addr_urlencoded: &str,
 ) -> Option<Vec<ServerParams>> {
-    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        &format!(
+            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
+            param_domain, param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 300);
 
-    let mut progress = 300;
-    for source in &sources {
-        let res = source.fetch(ctx, param).await;
-        progress!(ctx, progress);
-        progress += 10;
-        if let Ok(res) = res {
-            return Some(res);
-        }
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+        &format!(
+            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
+            &param_domain, &param_addr_urlencoded
+        ),
+        param,
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 310);
+
+    // Outlook uses always SSL but different domains (this comment describes the next two steps)
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 320);
+
+    if let Ok(res) = outlk_autodiscover(
+        ctx,
+        format!(
+            "https://autodiscover.{}/autodiscover/autodiscover.xml",
+            &param_domain
+        ),
+    )
+    .await
+    {
+        return Some(res);
+    }
+    progress!(ctx, 330);
+
+    // always SSL for Thunderbird's database
+    if let Ok(res) = moz_autoconfigure(
+        ctx,
+        &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
+        param,
+    )
+    .await
+    {
+        return Some(res);
     }
 
     None
 }
 
-fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
-    info!(
-        context,
-        "checking internal provider-info for offline autoconfig"
-    );
-
-    if let Some(provider) = provider::get_provider_info(&addr) {
-        match provider.status {
-            provider::Status::OK | provider::Status::PREPARATION => {
-                if provider.server.is_empty() {
-                    info!(context, "offline autoconfig found, but no servers defined");
-                    None
-                } else {
-                    info!(context, "offline autoconfig found");
-                    let servers = provider
-                        .server
-                        .iter()
-                        .map(|s| ServerParams {
-                            protocol: s.protocol,
-                            socket: s.socket,
-                            hostname: s.hostname.to_string(),
-                            port: s.port,
-                            username: match s.username_pattern {
-                                UsernamePattern::EMAIL => addr.to_string(),
-                                UsernamePattern::EMAILLOCALPART => {
-                                    if let Some(at) = addr.find('@') {
-                                        addr.split_at(at).0.to_string()
-                                    } else {
-                                        addr.to_string()
-                                    }
-                                }
-                            },
-                        })
-                        .collect();
-                    Some(servers)
-                }
-            }
-            provider::Status::BROKEN => {
-                info!(context, "offline autoconfig found, provider is broken");
-                None
-            }
-        }
-    } else {
-        info!(context, "no offline autoconfig found");
-        None
-    }
-}
-
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     imap: &mut Imap,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -515,7 +528,10 @@ async fn try_imap_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+    if let Err(err) = imap
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -532,6 +548,7 @@ async fn try_smtp_one_param(
     param: &ServerLoginParam,
     addr: &str,
     oauth2: bool,
+    provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
@@ -540,7 +557,10 @@ async fn try_smtp_one_param(
     );
     info!(context, "Trying: {}", inf);
 
-    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+    if let Err(err) = smtp
+        .connect(context, param, addr, oauth2, provider_strict_tls)
+        .await
+    {
         info!(context, "failure: {}", err);
         Err(ConfigurationError {
             config: inf,
@@ -573,10 +593,7 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
         .iter()
         .all(|e| e.msg.to_lowercase().contains("could not resolve"))
     {
-        return context
-            .stock_str(StockMessage::ErrorNoNetwork)
-            .await
-            .to_string();
+        return stock_str::error_no_network(context).await;
     }
 
     if errors.iter().all(|e| e.msg == first_err.msg) {
@@ -609,37 +626,16 @@ pub enum Error {
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
-    use super::*;
-    use crate::config::*;
-    use crate::test_utils::*;
+    use crate::config::Config;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_no_panic_on_bad_credentials() {
         let t = TestContext::new().await;
-        t.ctx
-            .set_config(Config::Addr, Some("probably@unexistant.addr"))
+        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
             .await
             .unwrap();
-        t.ctx
-            .set_config(Config::MailPw, Some("123456"))
-            .await
-            .unwrap();
-        assert!(t.ctx.configure().await.is_err());
-    }
-
-    #[async_std::test]
-    async fn test_get_offline_autoconfig() {
-        let context = TestContext::new().await.ctx;
-
-        let addr = "someone123@example.org";
-        assert!(get_offline_autoconfig(&context, addr).is_none());
-
-        let addr = "someone123@nauta.cu";
-        let found_params = get_offline_autoconfig(&context, addr).unwrap();
-        assert_eq!(found_params.len(), 2);
-        assert_eq!(found_params[0].protocol, Protocol::IMAP);
-        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
-        assert_eq!(found_params[1].protocol, Protocol::SMTP);
-        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
+        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
+        assert!(t.configure().await.is_err());
     }
 }

src/configure/auto_mozilla.rs

@@ -115,8 +115,8 @@ fn parse_server<B: BufRead>(
                     MozConfigTag::Username => username = Some(val),
                     MozConfigTag::Sockettype => {
                         sockettype = match val.to_lowercase().as_ref() {
-                            "ssl" => Socket::SSL,
-                            "starttls" => Socket::STARTTLS,
+                            "ssl" => Socket::Ssl,
+                            "starttls" => Socket::Starttls,
                             "plain" => Socket::Plain,
                             _ => Socket::Automatic,
                         }
@@ -233,8 +233,8 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
         .chain(moz_ac.outgoing_servers.into_iter())
         .filter_map(|server| {
             let protocol = match server.typ.as_ref() {
-                "imap" => Some(Protocol::IMAP),
-                "smtp" => Some(Protocol::SMTP),
+                "imap" => Some(Protocol::Imap),
+                "smtp" => Some(Protocol::Smtp),
                 _ => None,
             };
             Some(ServerParams {
@@ -276,10 +276,10 @@ mod tests {
     fn test_parse_outlook_autoconfig() {
         let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
         let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res[0].protocol, Protocol::IMAP);
+        assert_eq!(res[0].protocol, Protocol::Imap);
         assert_eq!(res[0].hostname, "outlook.office365.com");
         assert_eq!(res[0].port, 993);
-        assert_eq!(res[1].protocol, Protocol::SMTP);
+        assert_eq!(res[1].protocol, Protocol::Smtp);
         assert_eq!(res[1].hostname, "smtp.office365.com");
         assert_eq!(res[1].port, 587);
     }
@@ -295,25 +295,25 @@ mod tests {
         assert_eq!(res.incoming_servers[0].typ, "imap");
         assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[0].port, 993);
-        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::Ssl);
         assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[1].typ, "imap");
         assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[1].port, 143);
-        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::Starttls);
         assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[2].typ, "pop3");
         assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[2].port, 995);
-        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::Ssl);
         assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[3].typ, "pop3");
         assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[3].port, 110);
-        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::Starttls);
         assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
 
         assert_eq!(res.outgoing_servers.len(), 1);
@@ -321,7 +321,7 @@ mod tests {
         assert_eq!(res.outgoing_servers[0].typ, "smtp");
         assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
         assert_eq!(res.outgoing_servers[0].port, 587);
-        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::Starttls);
         assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -73,7 +73,7 @@ fn parse_protocol<B: BufRead>(
                 }
             }
             Event::Text(ref e) => {
-                let val = e.unescape_and_decode(&reader).unwrap_or_default();
+                let val = e.unescape_and_decode(reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
@@ -117,7 +117,7 @@ fn parse_redirecturl<B: BufRead>(
     let mut buf = Vec::new();
     match reader.read_event(&mut buf)? {
         Event::Text(ref e) => {
-            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            let val = e.unescape_and_decode(reader).unwrap_or_default();
             Ok(val.trim().to_string())
         }
         _ => Ok("".to_string()),
@@ -154,7 +154,7 @@ fn parse_xml_reader<B: BufRead>(
 }
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
@@ -169,8 +169,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
         .filter_map(|protocol| {
             Some(ServerParams {
                 protocol: match protocol.typ.to_lowercase().as_ref() {
-                    "imap" => Some(Protocol::IMAP),
-                    "smtp" => Some(Protocol::SMTP),
+                    "imap" => Some(Protocol::Imap),
+                    "smtp" => Some(Protocol::Smtp),
                     _ => None,
                 }?,
                 socket: match protocol.ssl {
@@ -187,9 +187,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
 
 pub(crate) async fn outlk_autodiscover(
     context: &Context,
-    url: &str,
+    mut url: String,
 ) -> Result<Vec<ServerParams>, Error> {
-    let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
         let xml_raw = read_url(context, &url).await?;

src/configure/server_params.rs

@@ -25,7 +25,7 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
@@ -42,15 +42,15 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
         if self.hostname.is_empty() {
             self.hostname = param_domain.to_string();
             res.push(self.clone());
 
             self.hostname = match self.protocol {
-                Protocol::IMAP => "imap.".to_string() + param_domain,
-                Protocol::SMTP => "smtp.".to_string() + param_domain,
+                Protocol::Imap => "imap.".to_string() + param_domain,
+                Protocol::Smtp => "smtp.".to_string() + param_domain,
             };
             res.push(self.clone());
 
@@ -62,17 +62,17 @@ impl ServerParams {
         res
     }
 
-    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+    fn expand_ports(mut self) -> Vec<ServerParams> {
         // Try to infer port from socket security.
         if self.port == 0 {
             self.port = match self.socket {
-                Socket::SSL => match self.protocol {
-                    Protocol::IMAP => 993,
-                    Protocol::SMTP => 465,
+                Socket::Ssl => match self.protocol {
+                    Protocol::Imap => 993,
+                    Protocol::Smtp => 465,
                 },
-                Socket::STARTTLS | Socket::Plain => match self.protocol {
-                    Protocol::IMAP => 143,
-                    Protocol::SMTP => 587,
+                Socket::Starttls | Socket::Plain => match self.protocol {
+                    Protocol::Imap => 143,
+                    Protocol::Smtp => 587,
                 },
                 Socket::Automatic => 0,
             }
@@ -85,27 +85,27 @@ impl ServerParams {
             // Try common secure combinations.
 
             // Try STARTTLS
-            self.socket = Socket::STARTTLS;
+            self.socket = Socket::Starttls;
             self.port = match self.protocol {
-                Protocol::IMAP => 143,
-                Protocol::SMTP => 587,
+                Protocol::Imap => 143,
+                Protocol::Smtp => 587,
             };
             res.push(self.clone());
 
             // Try TLS
-            self.socket = Socket::SSL;
+            self.socket = Socket::Ssl;
             self.port = match self.protocol {
-                Protocol::IMAP => 993,
-                Protocol::SMTP => 465,
+                Protocol::Imap => 993,
+                Protocol::Smtp => 465,
             };
             res.push(self);
         } else if self.socket == Socket::Automatic {
             // Try TLS over user-provided port.
-            self.socket = Socket::SSL;
+            self.socket = Socket::Ssl;
             res.push(self.clone());
 
             // Try STARTTLS over user-provided port.
-            self.socket = Socket::STARTTLS;
+            self.socket = Socket::Starttls;
             res.push(self);
         } else {
             res.push(self);
@@ -140,10 +140,10 @@ mod tests {
     fn test_expand_param_vector() {
         let v = expand_param_vector(
             vec![ServerParams {
-                protocol: Protocol::IMAP,
+                protocol: Protocol::Imap,
                 hostname: "example.net".to_string(),
                 port: 0,
-                socket: Socket::SSL,
+                socket: Socket::Ssl,
                 username: "foobar".to_string(),
             }],
             "foobar@example.net",
@@ -153,12 +153,44 @@ mod tests {
         assert_eq!(
             v,
             vec![ServerParams {
-                protocol: Protocol::IMAP,
+                protocol: Protocol::Imap,
                 hostname: "example.net".to_string(),
                 port: 993,
-                socket: Socket::SSL,
+                socket: Socket::Ssl,
                 username: "foobar".to_string(),
             }],
         );
+
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::Smtp,
+                hostname: "example.net".to_string(),
+                port: 123,
+                socket: Socket::Automatic,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Ssl,
+                    username: "foobar".to_string()
+                },
+                ServerParams {
+                    protocol: Protocol::Smtp,
+                    hostname: "example.net".to_string(),
+                    port: 123,
+                    socket: Socket::Starttls,
+                    username: "foobar".to_string()
+                }
+            ],
+        );
     }
 }

src/constants.rs

@@ -1,8 +1,10 @@
 //! # Constants
-use deltachat_derive::*;
+use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
+use crate::chat::ChatId;
+
 pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
 #[derive(
@@ -19,7 +21,7 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
     Serialize,
     Deserialize,
 )]
-#[repr(u8)]
+#[repr(i8)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
@@ -32,7 +34,9 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -46,7 +50,9 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -59,7 +65,9 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -73,7 +81,9 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
+)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -99,9 +109,10 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
+pub const DC_GCM_INFO_ONLY: u32 = 0x02;
 
-pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
-pub const DC_GCL_ADD_SELF: usize = 0x02;
+pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01;
+pub const DC_GCL_ADD_SELF: u32 = 0x02;
 
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
@@ -113,15 +124,15 @@ pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
 
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub const DC_CHAT_ID_DEADDROP: u32 = 1;
+pub const DC_CHAT_ID_DEADDROP: ChatId = ChatId::new(1);
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
-pub const DC_CHAT_ID_TRASH: u32 = 3;
+pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3);
 /// only an indicator in a chatlist
-pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
+pub const DC_CHAT_ID_ARCHIVED_LINK: ChatId = ChatId::new(6);
 /// only an indicator in a chatlist
-pub const DC_CHAT_ID_ALLDONE_HINT: u32 = 7;
+pub const DC_CHAT_ID_ALLDONE_HINT: ChatId = ChatId::new(7);
 /// larger chat IDs are "real" chats, their messages are "real" messages.
-pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
+pub const DC_CHAT_ID_LAST_SPECIAL: ChatId = ChatId::new(9);
 
 #[derive(
     Debug,
@@ -143,6 +154,7 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
+    Mailinglist = 140,
 }
 
 impl Default for Chattype {
@@ -155,8 +167,34 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// approx. max. length returned by dc_msg_get_text()
-pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
+/// string that indicates sth. is left out or truncated
+pub const DC_ELLIPSE: &str = "[...]";
+
+/// to keep bubbles and chat flow usable,
+/// and to avoid problems with controls using very long texts,
+/// we limit the text length to DC_DESIRED_TEXT_LEN.
+/// if the text is longer, the full text can be retrieved usind has_html()/get_html().
+///
+/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
+/// (a bit less as truncation may not be exact and ellipses may be added).
+///
+/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
+/// define max. number of bytes, _not_ unicode graphemes.
+/// in general, that seems to be okay for such an upper limit,
+/// esp. as calculating the number of graphemes is not simple
+/// (one graphemes may be a sequence of code points which is a sequence of bytes).
+/// also even if we have the exact number of graphemes,
+/// that would not always help on getting an idea about the screen space used
+/// (to keep bubbles and chat flow usable).
+///
+/// therefore, the number of bytes is only a very rough estimation,
+/// however, the ~30K seems to work okayish for a while,
+/// if it turns out, it is too few for some alphabet, we can still increase.
+pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
+
+/// approx. max. length (number of bytes) returned by dc_msg_get_text()
+pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
+
 /// approx. max. length returned by dc_get_msg_info()
 pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
@@ -196,7 +234,8 @@ pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
-pub const AVATAR_SIZE: u32 = 192;
+pub const BALANCED_AVATAR_SIZE: u32 = 256;
+pub const WORSE_AVATAR_SIZE: u32 = 128;
 
 // max. width/height of images
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
@@ -205,6 +244,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
+// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks.
+// this does not affect MIME'e `To:` header.
+// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
+pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
 #[derive(
     Debug,
     Display,
@@ -219,7 +263,7 @@ pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
     Serialize,
     Deserialize,
 )]
-#[repr(i32)]
+#[repr(u32)]
 pub enum Viewtype {
     Unknown = 0,
 
@@ -278,16 +322,6 @@ impl Default for Viewtype {
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-}
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -296,3 +330,100 @@ pub enum KeyType {
     Public = 0,
     Private = 1,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use num_traits::FromPrimitive;
+
+    #[test]
+    fn test_derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+
+    #[test]
+    fn test_viewtype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Viewtype::Unknown, Viewtype::default());
+        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
+        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
+        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
+        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
+        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
+        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
+        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
+        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
+        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
+        assert_eq!(
+            Viewtype::VideochatInvitation,
+            Viewtype::from_i32(70).unwrap()
+        );
+    }
+
+    #[test]
+    fn test_chattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Chattype::Undefined, Chattype::default());
+        assert_eq!(Chattype::Undefined, Chattype::from_i32(0).unwrap());
+        assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
+        assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
+        assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
+    }
+
+    #[test]
+    fn test_keygentype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyGenType::Default, KeyGenType::default());
+        assert_eq!(KeyGenType::Default, KeyGenType::from_i32(0).unwrap());
+        assert_eq!(KeyGenType::Rsa2048, KeyGenType::from_i32(1).unwrap());
+        assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_keytype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap());
+        assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_showemails_values() {
+        // values may be written to disk and must not change
+        assert_eq!(ShowEmails::Off, ShowEmails::default());
+        assert_eq!(ShowEmails::Off, ShowEmails::from_i32(0).unwrap());
+        assert_eq!(
+            ShowEmails::AcceptedContacts,
+            ShowEmails::from_i32(1).unwrap()
+        );
+        assert_eq!(ShowEmails::All, ShowEmails::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_blocked_values() {
+        // values may be written to disk and must not change
+        assert_eq!(Blocked::Not, Blocked::default());
+        assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
+        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
+        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
+    }
+
+    #[test]
+    fn test_mediaquality_values() {
+        // values may be written to disk and must not change
+        assert_eq!(MediaQuality::Balanced, MediaQuality::default());
+        assert_eq!(MediaQuality::Balanced, MediaQuality::from_i32(0).unwrap());
+        assert_eq!(MediaQuality::Worse, MediaQuality::from_i32(1).unwrap());
+    }
+
+    #[test]
+    fn test_videochattype_values() {
+        // values may be written to disk and must not change
+        assert_eq!(VideochatType::Unknown, VideochatType::default());
+        assert_eq!(VideochatType::Unknown, VideochatType::from_i32(0).unwrap());
+        assert_eq!(
+            VideochatType::BasicWebrtc,
+            VideochatType::from_i32(1).unwrap()
+        );
+        assert_eq!(VideochatType::Jitsi, VideochatType::from_i32(2).unwrap());
+    }
+}

src/context.rs

@@ -3,25 +3,28 @@
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
+use std::time::{Instant, SystemTime};
 
-use async_std::path::{Path, PathBuf};
-use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
-use async_std::task;
+use anyhow::{bail, ensure, Result};
+use async_std::{
+    channel::{self, Receiver, Sender},
+    path::{Path, PathBuf},
+    sync::{Arc, Mutex, RwLock},
+    task,
+};
 
-use crate::chat::*;
+use crate::chat::{get_chat_cnt, ChatId};
 use crate::config::Config;
-use crate::constants::*;
-use crate::contact::*;
-use crate::dc_tools::duration_to_str;
-use crate::error::*;
+use crate::constants::DC_VERSION_STR;
+use crate::contact::Contact;
+use crate::dc_tools::{duration_to_str, time};
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{self, MsgId};
+use crate::message::{self, MessageState, MsgId};
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -44,7 +47,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<Bob>,
+    pub(crate) bob: Bob,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -59,7 +62,12 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Id for this context on the current device.
+    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+
+    /// ID for this `Context` in the current process.
+    ///
+    /// This allows for multiple `Context`s open in a single process where each context can
+    /// be identified by this ID.
     pub(crate) id: u32,
 
     creation_time: SystemTime,
@@ -83,6 +91,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
+    res.insert("num_cpus", num_cpus::get().to_string());
     res.insert("level", "awesome".into());
     res
 }
@@ -121,7 +130,7 @@ impl Context {
             os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            bob: RwLock::new(Default::default()),
+            bob: Default::default(),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -131,6 +140,7 @@ impl Context {
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
+            last_full_folder_scan: Mutex::new(None),
         };
 
         let ctx = Context {
@@ -144,7 +154,7 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        if self.is_io_running().await {
+        if self.inner.is_io_running().await {
             info!(self, "IO is already running");
             return;
         }
@@ -155,18 +165,9 @@ impl Context {
         }
     }
 
-    /// Returns if the IO scheduler is running.
-    pub async fn is_io_running(&self) -> bool {
-        self.inner.is_io_running().await
-    }
-
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
-        if !self.is_io_running().await {
-            info!(self, "IO is not running");
-            return;
-        }
 
         self.inner.stop_io().await;
     }
@@ -197,7 +198,10 @@ impl Context {
         });
     }
 
-    /// Get the next queued event.
+    /// Returns a receiver for emitted events.
+    ///
+    /// Multiple emitters can be created, but note that in this case each emitted event will
+    /// only be received by one of the emitters, not by all of them.
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
@@ -219,7 +223,7 @@ impl Context {
 
         s.ongoing_running = true;
         s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel(1);
+        let (sender, receiver) = channel::bounded(1);
         s.cancel_sender = Some(sender);
 
         Ok(receiver)
@@ -246,7 +250,9 @@ impl Context {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
         if let Some(cancel) = s.cancel_sender.take() {
-            cancel.send(()).await;
+            if let Err(err) = cancel.send(()).await {
+                warn!(self, "could not cancel ongoing: {:?}", err);
+            }
         }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
@@ -265,64 +271,68 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await;
-        let l2 = LoginParam::from_database(self, "configured_").await;
-        let displayname = self.get_config(Config::Displayname).await;
-        let chats = get_chat_cnt(self).await as usize;
+        let l = LoginParam::from_database(self, "").await?;
+        let l2 = LoginParam::from_database(self, "configured_").await?;
+        let displayname = self.get_config(Config::Displayname).await?;
+        let chats = get_chat_cnt(self).await? as usize;
         let real_msgs = message::get_real_msg_cnt(self).await as usize;
         let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
-        let contacts = Contact::get_real_cnt(self).await as usize;
-        let is_configured = self.get_config_int(Config::Configured).await;
+        let contacts = Contact::get_real_cnt(self).await? as usize;
+        let is_configured = self.get_config_int(Config::Configured).await?;
         let dbversion = self
             .sql
-            .get_raw_config_int(self, "dbversion")
-            .await
+            .get_raw_config_int("dbversion")
+            .await?
             .unwrap_or_default();
         let journal_mode = self
             .sql
-            .query_get_value(self, "PRAGMA journal_mode;", paramsv![])
-            .await
+            .query_get_value("PRAGMA journal_mode;", paramsv![])
+            .await?
             .unwrap_or_else(|| "unknown".to_string());
-        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await;
-        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await;
-        let bcc_self = self.get_config_int(Config::BccSelf).await;
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
+        let bcc_self = self.get_config_int(Config::BccSelf).await?;
 
-        let prv_key_cnt: Option<isize> = self
+        let prv_key_cnt = self
             .sql
-            .query_get_value(self, "SELECT COUNT(*) FROM keypairs;", paramsv![])
-            .await;
+            .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
+            .await?;
 
-        let pub_key_cnt: Option<isize> = self
+        let pub_key_cnt = self
             .sql
-            .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
-            .await;
+            .count("SELECT COUNT(*) FROM acpeerstates;", paramsv![])
+            .await?;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
             Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch).await;
-        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
-        let mvbox_move = self.get_config_int(Config::MvboxMove).await;
+        let inbox_watch = self.get_config_int(Config::InboxWatch).await?;
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await?;
+        let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
+        let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
         let folders_configured = self
             .sql
-            .get_raw_config_int(self, "folders_configured")
-            .await
+            .get_raw_config_int("folders_configured")
+            .await?
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
             .get_config(Config::ConfiguredSentboxFolder)
-            .await
+            .await?
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
             .get_config(Config::ConfiguredMvboxFolder)
-            .await
+            .await?
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
+
+        // insert values
+        res.insert("bot", self.get_config_int(Config::Bot).await?.to_string());
         res.insert("number_of_chats", chats.to_string());
         res.insert("number_of_chat_messages", real_msgs.to_string());
         res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
@@ -335,41 +345,90 @@ impl Context {
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
-                .await
+                .await?
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
+        res.insert(
+            "fetch_existing_msgs",
+            self.get_config_int(Config::FetchExistingMsgs)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "show_emails",
+            self.get_config_int(Config::ShowEmails).await?.to_string(),
+        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
+        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert(
+            "key_gen_type",
+            self.get_config_int(Config::KeyGenType).await?.to_string(),
+        );
         res.insert("bcc_self", bcc_self.to_string());
-        res.insert(
-            "private_key_count",
-            prv_key_cnt.unwrap_or_default().to_string(),
-        );
-        res.insert(
-            "public_key_count",
-            pub_key_cnt.unwrap_or_default().to_string(),
-        );
+        res.insert("private_key_count", prv_key_cnt.to_string());
+        res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);
+        res.insert(
+            "webrtc_instance",
+            self.get_config(Config::WebrtcInstance)
+                .await?
+                .unwrap_or_else(|| "<unset>".to_string()),
+        );
+        res.insert(
+            "media_quality",
+            self.get_config_int(Config::MediaQuality).await?.to_string(),
+        );
+        res.insert(
+            "delete_device_after",
+            self.get_config_int(Config::DeleteDeviceAfter)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "delete_server_after",
+            self.get_config_int(Config::DeleteServerAfter)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "last_housekeeping",
+            self.get_config_int(Config::LastHousekeeping)
+                .await?
+                .to_string(),
+        );
+        res.insert(
+            "scan_all_folders_debounce_secs",
+            self.get_config_int(Config::ScanAllFoldersDebounceSecs)
+                .await?
+                .to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
 
-        res
+        Ok(res)
     }
 
-    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop: i32 = 0;
-        self.sql
+    /// Get a list of fresh, unmuted messages in any chat but deaddrop.
+    ///
+    /// The list starts with the most recent message
+    /// and is typically used to show notifications.
+    /// Moreover, the number of returned messages
+    /// can be used for a badge counter on the app icon.
+    pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
+        let list = self
+            .sql
             .query_map(
                 concat!(
                     "SELECT m.id",
@@ -380,67 +439,41 @@ impl Context {
                     "        ON m.chat_id=c.id",
                     " WHERE m.state=?",
                     "   AND m.hidden=0",
-                    "   AND m.chat_id>?",
+                    "   AND m.chat_id>9",
                     "   AND ct.blocked=0",
-                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    "   AND c.blocked=0",
+                    "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
                 ),
-                paramsv![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
+                paramsv![MessageState::InFresh, time()],
                 |row| row.get::<_, MsgId>(0),
                 |rows| {
-                    let mut ret = Vec::new();
+                    let mut list = Vec::new();
                     for row in rows {
-                        ret.push(row?);
+                        list.push(row?);
                     }
-                    Ok(ret)
+                    Ok(list)
                 },
             )
-            .await
-            .unwrap_or_default()
+            .await?;
+        Ok(list)
     }
 
-    #[allow(non_snake_case)]
-    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
-        let real_query = query.as_ref().trim();
+    /// Searches for messages containing the query string.
+    ///
+    /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
+    /// is `None` this searches messages from all chats.
+    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: &str) -> Result<Vec<MsgId>> {
+        let real_query = query.trim();
         if real_query.is_empty() {
-            return Vec::new();
+            return Ok(Vec::new());
         }
-        let strLikeInText = format!("%{}%", real_query);
-        let strLikeBeg = format!("{}%", real_query);
+        let str_like_in_text = format!("%{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " WHERE m.chat_id=?",
-                "   AND m.hidden=0",
-                "   AND ct.blocked=0",
-                "   AND (txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp,m.id;"
-            )
-        } else {
-            concat!(
-                "SELECT m.id AS id, m.timestamp AS timestamp",
-                " FROM msgs m",
-                " LEFT JOIN contacts ct",
-                "        ON m.from_id=ct.id",
-                " LEFT JOIN chats c",
-                "        ON m.chat_id=c.id",
-                " WHERE m.chat_id>9",
-                "   AND m.hidden=0",
-                "   AND (c.blocked=0 OR c.blocked=?)",
-                "   AND ct.blocked=0",
-                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
-                " ORDER BY m.timestamp DESC,m.id DESC;"
-            )
-        };
-
-        self.sql
-            .query_map(
+        let do_query = |query, params| {
+            self.sql.query_map(
                 query,
-                paramsv![chat_id, strLikeInText, strLikeBeg],
+                params,
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -450,23 +483,75 @@ impl Context {
                     Ok(ret)
                 },
             )
-            .await
-            .unwrap_or_default()
+        };
+
+        let list = if let Some(chat_id) = chat_id {
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 WHERE m.chat_id=?
+                   AND m.hidden=0
+                   AND ct.blocked=0
+                   AND txt LIKE ?
+                 ORDER BY m.timestamp,m.id;",
+                paramsv![chat_id, str_like_in_text],
+            )
+            .await?
+        } else {
+            // For performance reasons results are sorted only by `id`, that is in the order of
+            // message reception.
+            //
+            // Unlike chat view, sorting by `timestamp` is not necessary but slows down the query by
+            // ~25% according to benchmarks.
+            //
+            // To speed up incremental search, where queries for few characters usually return lots
+            // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
+            // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
+            // The limit is documented and UI may add a hint when getting 1000 results.
+            do_query(
+                "SELECT m.id AS id, m.timestamp AS timestamp
+                 FROM msgs m
+                 LEFT JOIN contacts ct
+                        ON m.from_id=ct.id
+                 LEFT JOIN chats c
+                        ON m.chat_id=c.id
+                 WHERE m.chat_id>9
+                   AND m.hidden=0
+                   AND c.blocked=0
+                   AND ct.blocked=0
+                   AND m.txt LIKE ?
+                 ORDER BY m.id DESC LIMIT 1000",
+                paramsv![str_like_in_text],
+            )
+            .await?
+        };
+
+        Ok(list)
     }
 
-    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredInboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub async fn is_inbox(&self, folder_name: &str) -> Result<bool> {
+        let inbox = self.get_config(Config::ConfiguredInboxFolder).await?;
+        Ok(inbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredSentboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub async fn is_sentbox(&self, folder_name: &str) -> Result<bool> {
+        let sentbox = self.get_config(Config::ConfiguredSentboxFolder).await?;
+
+        Ok(sentbox.as_deref() == Some(folder_name))
     }
 
-    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredMvboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
+        let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
+
+        Ok(mvbox.as_deref() == Some(folder_name))
+    }
+
+    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
+        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
+
+        Ok(spam.as_deref() == Some(folder_name))
     }
 
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
@@ -483,14 +568,19 @@ impl InnerContext {
     }
 
     async fn stop_io(&self) {
-        assert!(self.is_io_running().await, "context is already stopped");
-        let token = {
-            let lock = &*self.scheduler.read().await;
-            lock.pre_stop().await
-        };
-        {
-            let lock = &mut *self.scheduler.write().await;
-            lock.stop(token).await;
+        if self.is_io_running().await {
+            let token = {
+                let lock = &*self.scheduler.read().await;
+                lock.pre_stop().await
+            };
+            {
+                let lock = &mut *self.scheduler.write().await;
+                lock.stop(token).await;
+            }
+        }
+
+        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
+            ephemeral_task.cancel().await;
         }
     }
 }
@@ -513,7 +603,16 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
+    use crate::chat::{
+        get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
+    };
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::dc_tools::dc_create_outgoing_rfc724_mid;
+    use crate::message::Message;
+    use crate::test_utils::TestContext;
+    use std::time::Duration;
+    use strum::IntoEnumIterator;
 
     #[async_std::test]
     async fn test_wrong_db() {
@@ -527,10 +626,146 @@ mod tests {
     #[async_std::test]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
-        let fresh = t.ctx.get_fresh_msgs().await;
+        let fresh = t.get_fresh_msgs().await.unwrap();
         assert!(fresh.is_empty())
     }
 
+    async fn receive_msg(t: &TestContext, chat: &Chat) {
+        let members = get_chat_contacts(t, chat.id).await.unwrap();
+        let contact = Contact::load_from_db(t, *members.first().unwrap())
+            .await
+            .unwrap();
+        let msg = format!(
+            "From: {}\n\
+             To: alice@example.com\n\
+             Message-ID: <{}>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+             \n\
+             hello\n",
+            contact.get_addr(),
+            dc_create_outgoing_rfc724_mid(None, contact.get_addr())
+        );
+        println!("{}", msg);
+        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
+            .await
+            .unwrap();
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_chats() {
+        // receive various mails in 3 chats
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        let claire = t.create_chat_with_contact("", "claire@g.it").await;
+        let dave = t.create_chat_with_contact("", "dave@g.it").await;
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.unwrap().len(), 1);
+        assert_eq!(bob.id.get_fresh_msg_cnt(&t).await.unwrap(), 1);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &claire).await;
+        assert_eq!(
+            get_chat_msgs(&t, claire.id, 0, None).await.unwrap().len(),
+            2
+        );
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
+
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(get_chat_msgs(&t, dave.id, 0, None).await.unwrap().len(), 3);
+        assert_eq!(dave.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
+
+        // mute one of the chats
+        set_muted(&t, claire.id, MuteDuration::Forever)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
+
+        // receive more messages
+        receive_msg(&t, &bob).await;
+        receive_msg(&t, &claire).await;
+        receive_msg(&t, &dave).await;
+        assert_eq!(
+            get_chat_msgs(&t, claire.id, 0, None).await.unwrap().len(),
+            3
+        );
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
+
+        // unmute claire again
+        set_muted(&t, claire.id, MuteDuration::NotMuted)
+            .await
+            .unwrap();
+        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
+    }
+
+    #[async_std::test]
+    async fn test_get_fresh_msgs_and_muted_until() {
+        let t = TestContext::new_alice().await;
+        let bob = t.create_chat_with_contact("", "bob@g.it").await;
+        receive_msg(&t, &bob).await;
+        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.unwrap().len(), 1);
+
+        // chat is unmuted by default, here and in the following assert(),
+        // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
+        // have the same view to the database.
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with mute_until in the future
+        set_muted(
+            &t,
+            bob.id,
+            MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
+        )
+        .await
+        .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with mute_until in the past,
+        // we need to modify the database directly
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=? WHERE id=?;",
+                paramsv![time() - 3600, bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+        // test get_fresh_msgs() with "forever" mute_until
+        set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+        // to test get_fresh_msgs() with invalid mute_until (everything < -1),
+        // that results in "muted forever" by definition.
+        t.sql
+            .execute(
+                "UPDATE chats SET muted_until=-2 WHERE id=?;",
+                paramsv![bob.id],
+            )
+            .await
+            .unwrap();
+        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+        assert!(!bob.is_muted());
+        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+    }
+
     #[async_std::test]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
@@ -586,14 +821,14 @@ mod tests {
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
-        std::mem::drop(t.ctx);
+        std::mem::drop(t);
     }
 
     #[async_std::test]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
-        let info = t.ctx.get_info().await;
+        let info = t.get_info().await.unwrap();
         assert!(info.get("database_dir").is_some());
     }
 
@@ -604,4 +839,137 @@ mod tests {
         assert!(info.get("database_dir").is_none());
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
+
+    #[async_std::test]
+    async fn test_get_info_completeness() {
+        // For easier debugging,
+        // get_info() shall return all important information configurable by the Config-values.
+        //
+        // There are exceptions for Config-values considered to be unimportant,
+        // too sensitive or summarized in another item.
+        let skip_from_get_info = vec![
+            "addr",
+            "displayname",
+            "imap_certificate_checks",
+            "mail_server",
+            "mail_user",
+            "mail_pw",
+            "mail_port",
+            "mail_security",
+            "notify_about_wrong_pw",
+            "save_mime_headers",
+            "selfstatus",
+            "send_server",
+            "send_user",
+            "send_pw",
+            "send_port",
+            "send_security",
+            "server_flags",
+            "smtp_certificate_checks",
+        ];
+        let t = TestContext::new().await;
+        let info = t.get_info().await.unwrap();
+        for key in Config::iter() {
+            let key: String = key.to_string();
+            if !skip_from_get_info.contains(&&*key)
+                && !key.starts_with("configured")
+                && !key.starts_with("sys.")
+            {
+                assert!(
+                    info.contains_key(&*key),
+                    "'{}' missing in get_info() output",
+                    key
+                );
+            }
+        }
+    }
+
+    #[async_std::test]
+    async fn test_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Global search finds nothing.
+        let res = alice.search_msgs(None, "foo").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds nothing.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert!(res.is_empty());
+
+        // Add messages to chat with Bob.
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.set_text(Some("foobar".to_string()));
+        send_msg(&alice, chat.id, &mut msg1).await?;
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.set_text(Some("barbaz".to_string()));
+        send_msg(&alice, chat.id, &mut msg2).await?;
+
+        // Global search with a part of text finds the message.
+        let res = alice.search_msgs(None, "ob").await?;
+        assert_eq!(res.len(), 1);
+
+        // Global search for "bar" matches both "foobar" and "barbaz".
+        let res = alice.search_msgs(None, "bar").await?;
+        assert_eq!(res.len(), 2);
+
+        // Message added later is returned first.
+        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.get(1), Some(&msg1.id));
+
+        // Global search with longer text does not find any message.
+        let res = alice.search_msgs(None, "foobarbaz").await?;
+        assert!(res.is_empty());
+
+        // Search for random string finds nothing.
+        let res = alice.search_msgs(None, "abc").await?;
+        assert!(res.is_empty());
+
+        // Search in chat with Bob finds the message.
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1);
+
+        // Search in Saved Messages does not find the message.
+        let res = alice.search_msgs(Some(self_talk), "foo").await?;
+        assert!(res.is_empty());
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_limit_search_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let chat = alice
+            .create_chat_with_contact("Bob", "bob@example.org")
+            .await;
+
+        // Add 999 messages
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foobar".to_string()));
+        for _ in 0..999 {
+            send_msg(&alice, chat.id, &mut msg).await?;
+        }
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 999);
+
+        // Add one more message, no limit yet
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // Add one more message, that one is truncated then
+        send_msg(&alice, chat.id, &mut msg).await?;
+        let res = alice.search_msgs(None, "foo").await?;
+        assert_eq!(res.len(), 1000);
+
+        // In-chat should not be not limited
+        let res = alice.search_msgs(Some(chat.id), "foo").await?;
+        assert_eq!(res.len(), 1001);
+
+        Ok(())
+    }
 }

src/dc_tools.rs

@@ -12,26 +12,24 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
+use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
-use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::message::Message;
 use crate::provider::get_provider_update_timestamp;
-use crate::stock::StockMessage;
+use crate::stock_str;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
-
     let count = buf.chars().count();
-    if approx_chars > 0 && count > approx_chars + ellipse.len() {
+    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -39,40 +37,15 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
         }
     } else {
         Cow::Borrowed(buf)
     }
 }
 
-/// the colors must fulfill some criterions as:
-/// - contrast to black and to white
-/// - work as a text-color
-/// - being noticeable on a typical map
-/// - harmonize together while being different enough
-/// (therefore, we cannot just use random rgb colors :)
-const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
-];
-
-#[allow(clippy::indexing_slicing)]
-pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
-    let str_lower = s.as_ref().to_lowercase();
-    let mut checksum = 0;
-    let bytes = str_lower.as_bytes();
-    for (i, byte) in bytes.iter().enumerate() {
-        checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
-    }
-    let color_index = checksum % COLORS.len();
-
-    COLORS[color_index]
-}
-
 /* ******************************************************************************
  * date/time tools
  ******************************************************************************/
@@ -169,15 +142,14 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
     if now < known_past_timestamp {
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some(
-            context
-                .stock_string_repl_str(
-                    StockMessage::BadTimeMsgBody,
-                    Local
-                        .timestamp(now, 0)
-                        .format("%Y-%m-%d %H:%M:%S")
-                        .to_string(),
-                )
-                .await,
+            stock_str::bad_time_msg_body(
+                context,
+                Local
+                    .timestamp(now, 0)
+                    .format("%Y-%m-%d %H:%M:%S")
+                    .to_string(),
+            )
+            .await,
         );
         add_device_msg_with_importance(
             context,
@@ -201,12 +173,7 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
 async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
     if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(
-            context
-                .stock_str(StockMessage::UpdateReminderMsgBody)
-                .await
-                .into(),
-        );
+        msg.text = Some(stock_str::update_reminder_msg_body(context).await);
         add_device_msg(
             context,
             Some(
@@ -537,34 +504,15 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn get_next_backup_path_old(
-    folder: impl AsRef<Path>,
-    backup_time: i64,
-) -> Result<PathBuf, Error> {
-    let folder = PathBuf::from(folder.as_ref());
-    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        .format("delta-chat-%Y-%m-%d")
-        .to_string();
-
-    // 64 backup files per day should be enough for everyone
-    for i in 0..64 {
-        let mut path = folder.clone();
-        path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists().await {
-            return Ok(path);
-        }
-    }
-    bail!("could not create backup file, disk full?");
-}
-
 /// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path_new(
+pub(crate) async fn get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<(PathBuf, PathBuf), Error> {
     let folder = PathBuf::from(folder.as_ref());
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
         .format("delta-chat-backup-%Y-%m-%d")
         .to_string();
 
@@ -692,17 +640,9 @@ impl rusqlite::types::ToSql for EmailAddress {
     }
 }
 
-/// Utility to check if a in the binary represantion of listflags
-/// the bit at position bitindex is 1.
-pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
-    let listflags = listflags as usize;
-    (listflags & bitindex) == bitindex
-}
-
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
-pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+pub(crate) fn improve_single_line_input(input: &str) -> String {
     input
-        .as_ref()
         .replace("\n", " ")
         .replace("\r", " ")
         .trim()
@@ -721,15 +661,33 @@ where
     }
 }
 
+pub fn remove_subject_prefix(last_subject: &str) -> String {
+    let subject_start = if last_subject.starts_with("Chat:") {
+        0
+    } else {
+        // "Antw:" is the longest abbreviation in
+        // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+        // so look at the first _5_ characters:
+        match last_subject.chars().take(5).position(|c| c == ':') {
+            Some(prefix_end) => prefix_end + 1,
+            None => 0,
+        }
+    };
+    last_subject
+        .chars()
+        .skip(subject_start)
+        .collect::<String>()
+        .trim()
+        .to_string()
+}
+
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use std::convert::TryInto;
 
-    use crate::constants::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_rust_ftoa() {
@@ -918,7 +876,7 @@ mod tests {
     #[async_std::test]
     async fn test_file_handling() {
         let t = TestContext::new().await;
-        let context = &t.ctx;
+        let context = &t;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
                 $ctx.get_blobdir()
@@ -928,16 +886,6 @@ mod tests {
         }
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
-        if dc_file_exist!(context, "$BLOBDIR/foobar").await
-            || dc_file_exist!(context, "$BLOBDIR/dada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
-        {
-            dc_delete_file(context, "$BLOBDIR/foobar").await;
-            dc_delete_file(context, "$BLOBDIR/dada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
-        }
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());
@@ -980,29 +928,15 @@ mod tests {
         assert!(!dc_file_exist!(context, &fn0).await);
     }
 
-    #[test]
-    fn test_listflags_has() {
-        let listflags: u32 = 0x1101;
-        assert!(listflags_has(listflags, 0x1));
-        assert!(!listflags_has(listflags, 0x10));
-        assert!(listflags_has(listflags, 0x100));
-        assert!(listflags_has(listflags, 0x1000));
-        let listflags: u32 = (DC_GCL_ADD_SELF | DC_GCL_VERIFIED_ONLY).try_into().unwrap();
-        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY));
-        assert!(listflags_has(listflags, DC_GCL_ADD_SELF));
-        let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
-        assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
-    }
-
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx).await,
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await,
+            dc_create_smeared_timestamp(&t).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -1014,13 +948,13 @@ mod tests {
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t, count as usize).await;
+        let next = dc_smeared_time(&t).await;
         assert!((start + count - 1) < next);
     }
 
@@ -1094,53 +1028,61 @@ mod tests {
             / 1_000;
 
         // a correct time must not add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // we cannot find out if a date in the future is wrong - a device message is not added
-        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // a date in the past must add a device message
-        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
         assert_eq!(msgs.len(), 1);
 
         // the message should be added only once a day - test that an hour later and nearly a day later
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
         assert_eq!(msgs.len(), 1);
 
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24 - 1,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
         assert_eq!(msgs.len(), 1);
 
         // next day, there should be another device message
         maybe_warn_on_bad_time(
-            &t.ctx,
+            &t,
             timestamp_past + 60 * 60 * 24,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         assert_eq!(device_chat_id, chats.get_chat_id(0));
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
         assert_eq!(msgs.len(), 2);
     }
 
@@ -1152,51 +1094,66 @@ mod tests {
         // in about 6 months, the app should not be outdated
         // (if this fails, provider-db is not updated since 6 months)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 180 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // in 1 year, the app should be considered as outdated
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + 365 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
         assert_eq!(msgs.len(), 1);
 
         // do not repeat the warning every day ...
+        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
         maybe_warn_on_outdated(
-            &t.ctx,
+            &t,
             timestamp_now + (365 + 1) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 1);
-        let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
-        assert_eq!(msgs.len(), 1);
-
-        // ... but every month
         maybe_warn_on_outdated(
-            &t.ctx,
-            timestamp_now + (365 + 31) * 24 * 60 * 60,
+            &t,
+            timestamp_now + (365 + 2) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
-        assert_eq!(msgs.len(), 2);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
+        let test_len = msgs.len();
+        assert!(test_len == 1 || test_len == 2);
+
+        // ... but every month
+        // (forward generous 33 days to avoid being in the same month as in the previous check)
+        maybe_warn_on_outdated(
+            &t,
+            timestamp_now + (365 + 33) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
+            .await
+            .unwrap();
+        assert_eq!(msgs.len(), test_len + 1);
     }
 }

src/dehtml.rs

@@ -2,8 +2,13 @@
 //!
 //! A module to remove HTML tags from the email text
 
+use std::io::BufRead;
+
 use once_cell::sync::Lazy;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use quick_xml::{
+    events::{BytesEnd, BytesStart, BytesText},
+    Reader,
+};
 
 static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
@@ -11,9 +16,40 @@ struct Dehtml {
     strbuilder: String,
     add_text: AddText,
     last_href: Option<String>,
+    /// GMX wraps a quote in `<div name="quote">`. After a `<div name="quote">`, this count is
+    /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
+    /// If this is > `0`, then we are inside a `<div name="quote">`
+    divs_since_quote_div: u32,
+    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
+    /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
+    divs_since_quoted_content_div: u32,
+    /// All-Inkl just puts the quote into `<blockquote> </blockquote>`. This count is
+    /// increased at each `<blockquote>` and decreased at each `</blockquote>`.
+    blockquotes_since_blockquote: u32,
 }
 
-#[derive(Debug, PartialEq)]
+impl Dehtml {
+    fn line_prefix(&self) -> &str {
+        if self.divs_since_quoted_content_div > 0 || self.blockquotes_since_blockquote > 0 {
+            "> "
+        } else {
+            ""
+        }
+    }
+    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
+        // line_end is e.g. "\n\n". We add "> " if necessary.
+        line_end.as_ref().to_owned() + self.line_prefix()
+    }
+    fn get_add_text(&self) -> AddText {
+        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
+            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
+        } else {
+            self.add_text
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
 enum AddText {
     No,
     YesRemoveLineEnds,
@@ -34,13 +70,16 @@ pub fn dehtml(buf: &str) -> Option<String> {
     None
 }
 
-pub fn dehtml_quick_xml(buf: &str) -> String {
+fn dehtml_quick_xml(buf: &str) -> String {
     let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
         add_text: AddText::YesRemoveLineEnds,
         last_href: None,
+        divs_since_quote_div: 0,
+        divs_since_quoted_content_div: 0,
+        blockquotes_since_blockquote: 0,
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -79,13 +118,16 @@ pub fn dehtml_quick_xml(buf: &str) -> String {
 }
 
 fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -93,13 +135,16 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.add_text == AddText::YesPreserveLineEnds
-        || dehtml.add_text == AddText::YesRemoveLineEnds
+    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
+        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.add_text == AddText::YesRemoveLineEnds {
+        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
+        } else if !dehtml.line_prefix().is_empty() {
+            let l = dehtml.append_prefix("\n");
+            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -110,8 +155,15 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        "div" => {
+            pop_tag(&mut dehtml.divs_since_quote_div);
+            pop_tag(&mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "a" => {
@@ -122,11 +174,16 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
+        "blockquote" => pop_tag(&mut dehtml.blockquotes_since_blockquote),
         _ => {}
     }
 }
@@ -139,19 +196,27 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "div" | "table" | "td" => {
-            dehtml.strbuilder += "\n\n";
+        "p" | "table" | "td" => {
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.add_text = AddText::YesRemoveLineEnds;
+        }
+        #[rustfmt::skip]
+        "div" => {
+            maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
+            maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
+
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "br" => {
-            dehtml.strbuilder += "\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n");
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "style" | "script" | "title" => {
             dehtml.add_text = AddText::No;
         }
         "pre" => {
-            dehtml.strbuilder += "\n\n";
+            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
@@ -172,15 +237,52 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             }
         }
         "b" | "strong" => {
-            dehtml.strbuilder += "*";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "*";
+            }
         }
         "i" | "em" => {
-            dehtml.strbuilder += "_";
+            if dehtml.get_add_text() != AddText::No {
+                dehtml.strbuilder += "_";
+            }
         }
+        "blockquote" => dehtml.blockquotes_since_blockquote += 1,
         _ => {}
     }
 }
 
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn pop_tag(count: &mut u32) {
+    if *count > 0 {
+        *count -= 1;
+    }
+}
+
+/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
+/// The `counts`s are stored in the `Dehtml` struct.
+fn maybe_push_tag(
+    event: &BytesStart,
+    reader: &Reader<impl BufRead>,
+    tag_name: &str,
+    count: &mut u32,
+) {
+    if *count > 0 || tag_contains_attr(event, reader, tag_name) {
+        *count += 1;
+    }
+}
+
+fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
+    event.attributes().any(|r| {
+        r.map(|a| {
+            a.unescape_and_decode_value(reader)
+                .map(|v| v == name)
+                .unwrap_or(false)
+        })
+        .unwrap_or(false)
+    })
+}
+
 pub fn dehtml_manually(buf: &str) -> String {
     // Just strip out everything between "<" and ">"
     let mut strbuilder = String::new();
@@ -212,6 +314,7 @@ mod tests {
                 "[ Foo ](https://example.com)",
             ),
             ("<b> bar </b>", "* bar *"),
+            ("<i>foo</i>", "_foo_"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
             // Despite missing ', this should be shown:
@@ -288,4 +391,24 @@ mod tests {
         let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
+
+    #[test]
+    fn test_pre_tag() {
+        let input = "<html><pre>\ntwo\nlines\n</pre></html>";
+        let txt = dehtml(input).unwrap();
+        assert_eq!(txt.trim(), "two\nlines");
+    }
+
+    #[async_std::test]
+    async fn test_quote_div() {
+        let input = include_str!("../test-data/message/gmx-quote-body.eml");
+        let dehtml = dehtml(input).unwrap();
+        println!("{}", dehtml);
+        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
+        assert_eq!(msg, "Test");
+        assert_eq!(forwarded, false);
+        assert_eq!(cut, false);
+        assert_eq!(top_quote.as_deref(), Some("test"));
+        assert_eq!(footer, None);
+    }
 }

src/e2ee.rs

@@ -2,18 +2,18 @@
 
 use std::collections::HashSet;
 
+use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
-use crate::aheader::*;
+use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
-use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
-use crate::keyring::*;
-use crate::peerstate::*;
+use crate::keyring::Keyring;
+use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
 #[derive(Debug)]
@@ -26,9 +26,9 @@ pub struct EncryptHelper {
 impl EncryptHelper {
     pub async fn new(context: &Context) -> Result<EncryptHelper> {
         let prefer_encrypt =
-            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await)
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await {
+        let addr = match context.get_config(Config::ConfiguredAddr).await? {
             None => {
                 bail!("addr not configured!");
             }
@@ -113,7 +113,7 @@ impl EncryptHelper {
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
+        peerstates: Vec<(Option<Peerstate>, &str)>,
     ) -> Result<String> {
         let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
@@ -153,7 +153,7 @@ pub async fn try_decrypt(
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -163,10 +163,10 @@ pub async fn try_decrypt(
     // Apply Autocrypt header
     if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(&header, message_time);
+            peerstate.apply_header(header, message_time);
             peerstate.save_to_db(&context.sql, false).await?;
         } else {
-            let p = Peerstate::from_header(context, header, message_time);
+            let p = Peerstate::from_header(header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
@@ -209,29 +209,58 @@ pub async fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
-/// Returns a reference to the encrypted payload and validates the autocrypt structure.
-fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
-    ensure!(
-        mail.ctype.mimetype == "multipart/encrypted",
-        "Not a multipart/encrypted message: {}",
-        mail.ctype.mimetype
-    );
+/// Returns a reference to the encrypted payload of a valid PGP/MIME message.
+///
+/// Returns `None` if the message is not a valid PGP/MIME message.
+fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+    if mail.ctype.mimetype != "multipart/encrypted" {
+        return None;
+    }
     if let [first_part, second_part] = &mail.subparts[..] {
-        ensure!(
-            first_part.ctype.mimetype == "application/pgp-encrypted",
-            "Invalid Autocrypt Level 1 version part: {:?}",
-            first_part.ctype,
-        );
-
-        ensure!(
-            second_part.ctype.mimetype == "application/octet-stream",
-            "Invalid Autocrypt Level 1 encrypted part: {:?}",
-            second_part.ctype
-        );
-
-        Ok(second_part)
+        if first_part.ctype.mimetype == "application/pgp-encrypted"
+            && second_part.ctype.mimetype == "application/octet-stream"
+        {
+            Some(second_part)
+        } else {
+            None
+        }
     } else {
-        bail!("Invalid Autocrypt Level 1 Mime Parts")
+        None
+    }
+}
+
+/// Returns a reference to the encrypted payload of a ["Mixed
+/// Up"][pgpmime-message-mangling] message.
+///
+/// According to [RFC 3156] encrypted messages should have
+/// `multipart/encrypted` MIME type and two parts, but Microsoft
+/// Exchange and ProtonMail IMAP/SMTP Bridge are known to mangle this
+/// structure by changing the type to `multipart/mixed` and prepending
+/// an empty part at the start.
+///
+/// ProtonMail IMAP/SMTP Bridge prepends a part literally saying
+/// "Empty Message", so we don't check its contents at all, checking
+/// only for `text/plain` type.
+///
+/// Returns `None` if the message is not a "Mixed Up" message.
+///
+/// [RFC 3156]: https://www.rfc-editor.org/info/rfc3156
+/// [pgpmime-message-mangling]: https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html
+fn get_mixed_up_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+    if mail.ctype.mimetype != "multipart/mixed" {
+        return None;
+    }
+    if let [first_part, second_part, third_part] = &mail.subparts[..] {
+        if first_part.ctype.mimetype == "text/plain"
+            && second_part.ctype.mimetype == "application/pgp-encrypted"
+            && third_part.ctype.mimetype == "application/octet-stream"
+        {
+            Some(third_part)
+        } else {
+            None
+        }
+    } else {
+        None
     }
 }
 
@@ -242,12 +271,12 @@ async fn decrypt_if_autocrypt_message(
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    let encrypted_data_part = match get_autocrypt_mime(mail) {
-        Err(_) => {
+    let encrypted_data_part = match get_autocrypt_mime(mail).or_else(|| get_mixed_up_mime(mail)) {
+        None => {
             // not an autocrypt mime message, abort and ignore
             return Ok(None);
         }
-        Ok(res) => res,
+        Some(res) => res,
     };
     info!(context, "Detected Autocrypt-mime message");
 
@@ -329,7 +358,7 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
-        .await
+        .await?
         .ok_or_else(|| {
             format_err!(concat!(
                 "Failed to get self address, ",
@@ -346,10 +375,10 @@ mod tests {
 
     use crate::chat;
     use crate::constants::Viewtype;
-    use crate::contact::{Contact, Origin};
     use crate::message::Message;
     use crate::param::Param;
-    use crate::test_utils::*;
+    use crate::peerstate::ToSave;
+    use crate::test_utils::{bob_keypair, TestContext};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -358,13 +387,13 @@ mod tests {
         async fn test_prexisting() {
             let t = TestContext::new().await;
             let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
+            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
             let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
+            assert!(ensure_secret_key_exists(&t).await.is_err());
         }
     }
 
@@ -411,27 +440,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[async_std::test]
-    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let (contact_alice_id, _modified) = Contact::add_or_lookup(
-            &bob.ctx,
-            "Alice",
-            "alice@example.com",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-        let (contact_bob_id, _modified) = Contact::add_or_lookup(
-            &alice.ctx,
-            "Bob",
-            "bob@example.net",
-            Origin::ManuallyCreated,
-        )
-        .await?;
-
-        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
-        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
@@ -512,14 +526,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         Ok(())
     }
 
-    fn new_peerstates(
-        ctx: &Context,
-        prefer_encrypt: EncryptPreference,
-    ) -> Vec<(Option<Peerstate<'_>>, &str)> {
+    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
         let addr = "bob@foo.bar";
         let pub_key = bob_keypair().public;
         let peerstate = Peerstate {
-            context: &ctx,
             addr: addr.into(),
             last_seen: 13,
             last_seen_autocrypt: 14,
@@ -534,36 +544,56 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             to_save: Some(ToSave::All),
             fingerprint_changed: false,
         };
-        let mut peerstates = Vec::new();
-        peerstates.push((Some(peerstate), addr));
-        peerstates
+        vec![(Some(peerstate), addr)]
     }
 
     #[async_std::test]
     async fn test_should_encrypt() {
         let t = TestContext::new_alice().await;
-        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+        let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
 
         // test with EncryptPreference::NoPreference:
         // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
-        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Reset
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with EncryptPreference::Mutual (self is also Mutual)
-        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
-        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = new_peerstates(EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with missing peerstate
-        let mut ps = Vec::new();
-        ps.push((None, "bob@foo.bar"));
-        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
-        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+        let ps = vec![(None, "bob@foo.bar")];
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+    }
+
+    #[test]
+    fn test_mixed_up_mime() -> Result<()> {
+        // "Mixed Up" mail as received when sending an encrypted
+        // message using Delta Chat Desktop via ProtonMail IMAP/SMTP
+        // Bridge.
+        let mixed_up_mime = include_bytes!("../test-data/message/protonmail-mixed-up.eml");
+        let mail = mailparse::parse_mail(mixed_up_mime)?;
+        assert!(get_autocrypt_mime(&mail).is_none());
+        assert!(get_mixed_up_mime(&mail).is_some());
+
+        // Same "Mixed Up" mail repaired by Thunderbird 78.9.0.
+        //
+        // It added `X-Enigmail-Info: Fixed broken PGP/MIME message`
+        // header although the repairing is done by the built-in
+        // OpenPGP support, not Enigmail.
+        let repaired_mime = include_bytes!("../test-data/message/protonmail-repaired.eml");
+        let mail = mailparse::parse_mail(repaired_mime)?;
+        assert!(get_autocrypt_mime(&mail).is_some());
+        assert!(get_mixed_up_mime(&mail).is_none());
+
+        Ok(())
     }
 }

src/ephemeral.rs

@@ -56,24 +56,26 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
-use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use anyhow::{ensure, Context as _, Result};
+use async_std::task;
+use serde::{Deserialize, Serialize};
+
+use crate::chat::{send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::error::{ensure, Error};
 use crate::events::EventType;
+use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::sql;
-use crate::stock::StockMessage;
-use async_std::task;
-use serde::{Deserialize, Serialize};
-use std::convert::{TryFrom, TryInto};
-use std::num::ParseIntError;
-use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use crate::stock_str;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -145,10 +147,10 @@ impl rusqlite::types::FromSql for Timer {
 
 impl ChatId {
     /// Get ephemeral message timer value in seconds.
-    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer> {
         let timer = context
             .sql
-            .query_get_value_result(
+            .query_get_value(
                 "SELECT ephemeral_timer FROM chats WHERE id=?;",
                 paramsv![self],
             )
@@ -164,7 +166,7 @@ impl ChatId {
         self,
         context: &Context,
         timer: Timer,
-    ) -> Result<(), Error> {
+    ) -> Result<()> {
         ensure!(!self.is_special(), "Invalid chat ID");
 
         context
@@ -187,7 +189,7 @@ impl ChatId {
     /// Set ephemeral message timer value in seconds.
     ///
     /// If timer value is 0, disable ephemeral message timer.
-    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<()> {
         if timer == self.get_ephemeral_timer(context).await? {
             return Ok(());
         }
@@ -211,29 +213,58 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     timer: Timer,
     from_id: u32,
 ) -> String {
-    let stock_message = match timer {
-        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+    match timer {
+        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
         Timer::Enabled { duration } => match duration {
-            60 => StockMessage::MsgEphemeralTimerMinute,
-            3600 => StockMessage::MsgEphemeralTimerHour,
-            86400 => StockMessage::MsgEphemeralTimerDay,
-            604_800 => StockMessage::MsgEphemeralTimerWeek,
-            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
-            _ => StockMessage::MsgEphemeralTimerEnabled,
+            0..=59 => {
+                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
+            }
+            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
+            61..=3599 => {
+                stock_str::msg_ephemeral_timer_minutes(
+                    context,
+                    format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
+            3601..=86399 => {
+                stock_str::msg_ephemeral_timer_hours(
+                    context,
+                    format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
+            86401..=604_799 => {
+                stock_str::msg_ephemeral_timer_days(
+                    context,
+                    format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
+            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
+            _ => {
+                stock_str::msg_ephemeral_timer_weeks(
+                    context,
+                    format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
+                    from_id,
+                )
+                .await
+            }
         },
-    };
-
-    context
-        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
-        .await
+    }
 }
 
 impl MsgId {
     /// Returns ephemeral message timer value for the message.
-    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
         let res = match context
             .sql
-            .query_get_value_result(
+            .query_get_value(
                 "SELECT ephemeral_timer FROM msgs WHERE id=?",
                 paramsv![self],
             )
@@ -246,7 +277,7 @@ impl MsgId {
     }
 
     /// Starts ephemeral message timer for the message if it is not started yet.
-    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
             let ephemeral_timestamp = time() + i64::from(duration);
 
@@ -273,30 +304,35 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
     let mut updated = context
         .sql
         .execute(
-            "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE \
-             ephemeral_timestamp != 0 \
-             AND ephemeral_timestamp < ? \
-             AND chat_id != ?",
+            // If you change which information is removed here, also change MsgId::trash() and
+            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
+            r#"
+UPDATE msgs
+SET 
+  chat_id=?, txt='', subject='', txt_raw='', 
+  mime_headers='', from_id=0, to_id=0, param=''
+WHERE
+  ephemeral_timestamp != 0
+  AND ephemeral_timestamp <= ?
+  AND chat_id != ?
+"#,
             paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
-        .await?
+        .await
+        .context("update failed")?
         > 0;
 
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .await
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .await
-            .unwrap_or_default()
-            .0;
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
+        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
+            .await?
+            .unwrap_or_default();
+        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
+            .await?
+            .unwrap_or_default();
 
         let threshold_timestamp = time() - delete_device_after;
 
@@ -321,7 +357,8 @@ pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, E
                     device_chat_id
                 ],
             )
-            .await?;
+            .await
+            .context("deleted update failed")?;
 
         updated |= rows_modified > 0;
     }
@@ -343,13 +380,15 @@ pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, E
 pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
-        .query_get_value_result(
-            "SELECT ephemeral_timestamp \
-         FROM msgs \
-         WHERE ephemeral_timestamp != 0 \
-           AND chat_id != ? \
-         ORDER BY ephemeral_timestamp ASC \
-         LIMIT 1",
+        .query_get_value(
+            r#"
+    SELECT ephemeral_timestamp
+    FROM msgs
+    WHERE ephemeral_timestamp != 0
+      AND chat_id != ?
+    ORDER BY ephemeral_timestamp ASC
+    LIMIT 1;
+    "#,
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
@@ -403,10 +442,10 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 ///
 /// It looks up the trash chat too, to find messages that are already
 /// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
-    let threshold_timestamp = match context.get_config_delete_server_after().await {
+    let threshold_timestamp = match context.get_config_delete_server_after().await? {
         None => 0,
         Some(delete_server_after) => now - delete_server_after,
     };
@@ -417,12 +456,16 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
             "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
+         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            paramsv![threshold_timestamp, now],
-            |row| row.get::<_, MsgId>(0),
+            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            |row| {
+                let msg_id: MsgId = row.get(0)?;
+                Ok(msg_id)
+            },
         )
         .await
 }
@@ -436,7 +479,7 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<O
 ///
 /// This function is supposed to be called in the background,
 /// e.g. from housekeeping task.
-pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
     context
         .sql
         .execute(
@@ -459,12 +502,19 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()>
 
 #[cfg(test)]
 mod tests {
+    use crate::param::Params;
+    use async_std::task::sleep;
+
     use super::*;
-    use crate::test_utils::*;
+    use crate::test_utils::TestContext;
+    use crate::{
+        chat::{self, Chat, ChatItem},
+        dc_tools::IsNoneOrEmpty,
+    };
 
     #[async_std::test]
     async fn test_stock_ephemeral_messages() {
-        let context = TestContext::new().await.ctx;
+        let context = TestContext::new().await;
 
         assert_eq!(
             stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
@@ -472,24 +522,78 @@ mod tests {
         );
 
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
-            "Message deletion timer is disabled."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 1 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
-            "Message deletion timer is set to 1 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 s by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
-            "Message deletion timer is set to 30 s."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 minute by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
-            "Message deletion timer is set to 1 minute."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 90 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 minutes by me."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
-            "Message deletion timer is set to 1 hour."
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 30 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 30 minutes by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 60 * 60 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1 hour by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled { duration: 5400 },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 1.5 hours by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 hours by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -497,10 +601,21 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 day."
+            "Message deletion timer is set to 1 day by me."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 2 * 24 * 60 * 60
+                },
+                DC_CONTACT_ID_SELF
+            )
+            .await,
+            "Message deletion timer is set to 2 days by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -508,10 +623,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 1 week."
+            "Message deletion timer is set to 1 week by me."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -519,10 +634,147 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                0
+                DC_CONTACT_ID_SELF
             )
             .await,
-            "Message deletion timer is set to 4 weeks."
+            "Message deletion timer is set to 4 weeks by me."
         );
     }
+
+    #[async_std::test]
+    async fn test_ephemeral_timer() -> anyhow::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let chat_alice = alice.create_chat(&bob).await.id;
+        let chat_bob = bob.create_chat(&alice).await.id;
+
+        // Alice sends message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        // Alice sends second message to Bob, with no timer
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Disabled
+        );
+
+        // Bob sets ephemeral timer and sends a message about timer change
+        chat_bob
+            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
+            .await?;
+        let sent_timer_change = bob.pop_sent_msg().await;
+
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Bob receives message from Alice.
+        // Alice message has no timer. However, Bob should not disable timer,
+        // because Alice replies to old message.
+        bob.recv_msg(&sent).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Disabled
+        );
+        assert_eq!(
+            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        // Alice receives message from Bob
+        alice.recv_msg(&sent_timer_change).await;
+
+        assert_eq!(
+            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+            Timer::Enabled { duration: 60 }
+        );
+
+        Ok(())
+    }
+
+    #[async_std::test]
+    async fn test_ephemeral_delete_msgs() {
+        let t = TestContext::new_alice().await;
+        let chat = t.get_self_chat().await;
+
+        t.send_text(chat.id, "Saved message, which we delete manually")
+            .await;
+        let msg = t.get_last_msg_in(chat.id).await;
+        msg.id.delete_from_db(&t).await.unwrap();
+        check_msg_was_deleted(&t, &chat, msg.id).await;
+
+        chat.id
+            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
+            .await
+            .unwrap();
+        let msg = t
+            .send_text(chat.id, "Saved message, disappearing after 1s")
+            .await;
+
+        sleep(Duration::from_millis(1100)).await;
+
+        // Check checks that the msg was deleted locally
+        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
+
+        // Check that the msg will be deleted on the server
+        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
+        t.sql
+            .execute(
+                "UPDATE msgs SET server_uid=1 WHERE id=?",
+                paramsv![msg.sender_msg_id],
+            )
+            .await
+            .unwrap();
+        let job = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(
+            job,
+            Some(job::Job::new(
+                job::Action::DeleteMsgOnImap,
+                msg.sender_msg_id.to_u32(),
+                Params::new(),
+                0,
+            ))
+        );
+        // Let's assume that executing the job fails on first try and the job is saved to the db
+        job.unwrap().save(&t).await.unwrap();
+
+        // Make sure that we don't get yet another job when loading from db
+        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
+        assert_eq!(job2, None);
+    }
+
+    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+        let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
+        // Check that the chat is empty except for possibly info messages:
+        for item in &chat_items {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
+                assert!(msg.is_info())
+            }
+        }
+
+        // Check that if there is a message left, the text and metadata are gone
+        if let Ok(msg) = Message::load_from_db(t, msg_id).await {
+            assert_eq!(msg.from_id, 0);
+            assert_eq!(msg.to_id, 0);
+            assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
+            let rawtxt: Option<String> = t
+                .sql
+                .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
+                .await
+                .unwrap();
+            assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
+        }
+    }
 }

src/error.rs

@@ -1,7 +1,5 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
-
 #[macro_export]
 macro_rules! ensure_eq {
     ($left:expr, $right:expr) => ({

src/events.rs

@@ -2,8 +2,8 @@
 
 use std::ops::Deref;
 
+use async_std::channel::{self, Receiver, Sender, TrySendError};
 use async_std::path::PathBuf;
-use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
@@ -18,7 +18,7 @@ pub struct Events {
 
 impl Default for Events {
     fn default() -> Self {
-        let (sender, receiver) = channel(1_000);
+        let (sender, receiver) = channel::bounded(1_000);
 
         Self { receiver, sender }
     }
@@ -35,7 +35,7 @@ impl Events {
                 // try again
                 self.emit(event);
             }
-            Err(TrySendError::Disconnected(_)) => {
+            Err(TrySendError::Closed(_)) => {
                 unreachable!("unable to emit event, channel disconnected");
             }
         }
@@ -47,6 +47,17 @@ impl Events {
     }
 }
 
+/// A receiver of events from a [`Context`].
+///
+/// See [`Context::get_event_emitter`] to create an instance.  If multiple instances are
+/// created events emitted by the [`Context`] will only be delivered to one of the
+/// `EventEmitter`s.
+///
+/// The `EventEmitter` is also a [`Stream`], so a typical usage is in a `while let` loop.
+///
+/// [`Context`]: crate::context::Context
+/// [`Context::get_event_emitter`]: crate::context::Context::get_event_emitter
+/// [`Stream`]: async_std::stream::Stream
 #[derive(Debug, Clone)]
 pub struct EventEmitter(Receiver<Event>);
 
@@ -73,9 +84,27 @@ impl async_std::stream::Stream for EventEmitter {
     }
 }
 
+/// The event emitted by a [`Context`] from an [`EventEmitter`].
+///
+/// Events are documented on the C/FFI API in `deltachat.h` as `DC_EVENT_*` contants.  The
+/// context emits them in relation to various operations happening, a lot of these are again
+/// documented in `deltachat.h`.
+///
+/// This struct [`Deref`]s to the [`EventType`].
+///
+/// [`Context`]: crate::context::Context
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Event {
+    /// The ID of the [`Context`] which emitted this event.
+    ///
+    /// This allows using multiple [`Context`]s in a single process as they are identified
+    /// by this ID.
+    ///
+    /// [`Context`]: crate::context::Context
     pub id: u32,
+    /// The event payload.
+    ///
+    /// These are documented in `deltachat.h` as the `DC_EVENT_*` constants.
     pub typ: EventType,
 }
 
@@ -88,7 +117,9 @@ impl Deref for Event {
 }
 
 impl EventType {
-    /// Returns the corresponding Event id.
+    /// Returns the corresponding Event ID.
+    ///
+    /// These are the IDs used in the `DC_EVENT_*` constants in `deltachat.h`.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
             .expect("missing id")
@@ -100,8 +131,9 @@ impl EventType {
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
 pub enum EventType {
     /// The library-user may write an informational string to the log.
-    /// Passed to the callback given to dc_context_new().
-    /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "100"))]
     Info(String),
 
@@ -134,14 +166,13 @@ pub enum EventType {
     DeletedBlobFile(String),
 
     /// The library-user should write a warning string to the log.
-    /// Passed to the callback given to dc_context_new().
     ///
-    /// This event should not be reported to the end-user using a popup or something like that.
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
     #[strum(props(id = "300"))]
     Warning(String),
 
     /// The library-user should report an error to the end-user.
-    /// Passed to the callback given to dc_context_new().
     ///
     /// As most things are asynchronous, things may go wrong at any time and the user
     /// should not be disturbed by a dialog or so.  Instead, use a bubble or so.
@@ -182,6 +213,8 @@ pub enum EventType {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
+    ///
+    /// The `chat_id` and `msg_id` values will be 0 if more than one message is changed.
     #[strum(props(id = "2000"))]
     MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 

src/export_chat.rs

@@ -0,0 +1,339 @@
+//! Export chats module
+//!
+//! ## Export Format
+//! The format of an exported chat is a zip file with the following structure:
+//! ```text
+//! ├── blobs/          # all files that are referenced by the chat
+//! ├── msg_info/
+//! │   └── [msg_id].txt # message info
+//! ├── msg_source/
+//! │   └── [msg_id].eml # email sourcecode of messages if availible¹
+//! └── chat.json        # chat info, messages and message authors
+//! ```
+//! ##### ¹ Saving Mime header
+//! To save the mime header you need to have the config option [`SaveMimeHeaders`] enabled.
+//! This option saves the mime headers on future messages. Normaly the original email source code is discarded to save space.
+//! You can use the repl tool to do this job:
+//! ```sh
+//! $ cargo run --example repl --features=repl /path/to/account/db.sqlite
+//! > set save_mime_headers 1
+//! ```
+//! [`SaveMimeHeaders`]: ../config/enum.Config.html#variant.SaveMimeHeaders
+
+use crate::chat::*;
+use crate::constants::Viewtype;
+use crate::constants::DC_GCM_ADDDAYMARKER;
+use crate::contact::*;
+use crate::context::Context;
+// use crate::error::Error;
+use crate::dc_tools::time;
+use crate::message::*;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::prelude::*;
+use std::path::Path;
+use zip::write::FileOptions;
+
+use crate::location::Location;
+use serde::Serialize;
+
+#[derive(Debug)]
+struct ExportChatResult {
+    chat_json: String,
+    // locations_geo_json: String,
+    message_ids: Vec<MsgId>,
+    referenced_blobs: Vec<String>,
+}
+
+pub async fn export_chat_to_zip(context: &Context, chat_id: ChatId, filename: &str) {
+    let res = export_chat_data(&context, chat_id).await;
+    let destination = std::path::Path::new(filename);
+    let pack_res = pack_exported_chat(&context, res, destination).await;
+    match &pack_res {
+        Ok(()) => println!("Exported chat successfully to {}", filename),
+        Err(err) => println!("Error {:?}", err),
+    };
+}
+
+async fn pack_exported_chat(
+    context: &Context,
+    artifact: ExportChatResult,
+    destination: &Path,
+) -> zip::result::ZipResult<()> {
+    let file = std::fs::File::create(&destination).unwrap();
+
+    let mut zip = zip::ZipWriter::new(file);
+
+    zip.start_file("chat.json", Default::default())?;
+    zip.write_all(artifact.chat_json.as_bytes())?;
+
+    zip.add_directory("blobs/", Default::default())?;
+
+    let options = FileOptions::default();
+    for blob_name in artifact.referenced_blobs {
+        let path = context.get_blobdir().join(&blob_name);
+
+        // println!("adding file {:?} as {:?} ...", path, &blob_name);
+        zip.start_file(format!("blobs/{}", &blob_name), options)?;
+        let mut f = File::open(path)?;
+
+        let mut buffer = Vec::new();
+        f.read_to_end(&mut buffer)?;
+        zip.write_all(&*buffer)?;
+        buffer.clear();
+    }
+
+    zip.add_directory("msg_info/", Default::default())?;
+    zip.add_directory("msg_source/", Default::default())?;
+    for id in artifact.message_ids {
+        zip.start_file(format!("msg_info/{}.txt", id.to_u32()), options)?;
+        zip.write_all((get_msg_info(&context, id).await).as_bytes())?;
+        if let Some(mime_headers) = get_mime_headers(&context, id).await {
+            zip.start_file(format!("msg_source/{}.eml", id.to_u32()), options)?;
+            zip.write_all((mime_headers).as_bytes())?;
+        }
+    }
+
+    zip.finish()?;
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct ChatJSON {
+    chat_json_version: u8,
+    export_timestamp: i64,
+    name: String,
+    color: String,
+    profile_img: Option<String>,
+    contacts: HashMap<u32, ContactJSON>,
+    referenced_external_messages:Vec<ChatItemJSON>,
+    messages: Vec<ChatItemJSON>,
+    locations: Vec<Location>,
+}
+
+#[derive(Serialize)]
+struct ContactJSON {
+    name: String,
+    email: String,
+    color: String,
+    profile_img: Option<String>,
+}
+
+#[derive(Serialize)]
+struct FileReference {
+    name: String,
+    filesize: u64,
+    mime: String,
+    path: String,
+}
+
+#[derive(Serialize)]
+struct Qoute {
+    quoted_text: String,
+    message_id: Option<u32>,
+}
+
+#[derive(Serialize)]
+#[serde(tag = "type")]
+enum ChatItemJSON {
+    Message {
+        id: u32,
+        author_id: u32, // from_id
+        view_type: Viewtype,
+        timestamp_sort: i64,
+        timestamp_sent: i64,
+        timestamp_rcvd: i64,
+        text: Option<String>,
+        attachment: Option<FileReference>,
+        location_id: Option<u32>,
+        is_info_message: bool,
+        show_padlock: bool,
+        state: MessageState,
+        is_forwarded: bool,
+        quote: Option<Qoute>
+    },
+    MessageError {
+        id: u32,
+        error: String,
+    },
+    DayMarker {
+        timestamp: i64,
+    },
+}
+
+impl ChatItemJSON {
+    pub async fn from_message(message: &Message, context: &Context) -> ChatItemJSON {
+        let msg_id = message.get_id();
+        ChatItemJSON::Message {
+            id: msg_id.to_u32(),
+            author_id: message.get_from_id(), // from_id
+            view_type: message.get_viewtype(),
+            timestamp_sort: message.timestamp_sort,
+            timestamp_sent: message.timestamp_sent,
+            timestamp_rcvd: message.timestamp_rcvd,
+            text: message.get_text(),
+            attachment: match message.get_file(context) {
+                Some(file) => Some(FileReference {
+                    name: message.get_filename().unwrap_or_else(|| "".to_owned()),
+                    filesize: message.get_filebytes(context).await,
+                    mime: message.get_filemime().unwrap_or_else(|| "".to_owned()),
+                    path: format!(
+                        "blobs/{}",
+                        file.file_name()
+                            .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                            .to_str()
+                            .unwrap()
+                    ),
+                }),
+                None => None,
+            },
+            location_id: match message.has_location() {
+                true => Some(message.location_id),
+                false => None,
+            },
+            is_info_message: message.is_info(),
+            show_padlock: message.get_showpadlock(),
+            state: message.get_state(),
+            is_forwarded: message.is_forwarded(),
+            quote: match message.quoted_text() {
+                Some(text) => match message.quoted_message(&context).await {
+                    Ok(Some(msg)) => Some(Qoute {
+                        quoted_text: text,
+                        message_id: Some(msg.get_id().to_u32())
+                    }),
+                    Err(_) | Ok(None) => Some(Qoute {
+                        quoted_text: text,
+                        message_id: None
+                    })
+                }
+                None => None
+            }
+        }
+    }
+}
+
+async fn export_chat_data(context: &Context, chat_id: ChatId) -> ExportChatResult {
+    let mut blobs = Vec::new();
+    let mut chat_author_ids = Vec::new();
+    // message_ids var is used for writing message info to files
+    let mut message_ids: Vec<MsgId> = Vec::new();
+    let mut message_json: Vec<ChatItemJSON> = Vec::new();
+    let mut referenced_external_messages: Vec<ChatItemJSON> = Vec::new();
+
+    for item in get_chat_msgs(context, chat_id, DC_GCM_ADDDAYMARKER, None).await {
+        if let Some(json_item) = match item {
+            ChatItem::Message { msg_id } => match Message::load_from_db(context, msg_id).await {
+                Ok(message) => {
+                    let filename = message.get_filename();
+                    if let Some(file) = filename {
+                        // push referenced blobs (attachments)
+                        blobs.push(file);
+                    }
+                    message_ids.push(message.id);
+                    // populate contactid list
+                    chat_author_ids.push(message.from_id);
+
+                    if let Ok(Some(ex_msg)) = message.quoted_message(&context).await {
+                        if ex_msg.get_chat_id() != chat_id {
+                            // if external add it to the file
+                            referenced_external_messages.push(ChatItemJSON::from_message(&ex_msg, &context).await)
+                            // contacts don't need to be referenced, because these should only be private replies
+                        }
+                    }
+
+                    Some(ChatItemJSON::from_message(&message, &context).await)
+                }
+                Err(error_message) => Some(ChatItemJSON::MessageError {
+                    id: msg_id.to_u32(),
+                    error: error_message.to_string(),
+                }),
+            },
+            ChatItem::DayMarker { timestamp } => Some(ChatItemJSON::DayMarker { timestamp }),
+            ChatItem::Marker1 => None,
+        } {
+            message_json.push(json_item)
+        }
+    }
+
+    // deduplicate contact list and load the contacts
+    chat_author_ids.sort();
+    chat_author_ids.dedup();
+    // load information about the authors
+    let mut chat_authors: HashMap<u32, ContactJSON> = HashMap::new();
+    chat_authors.insert(
+        0,
+        ContactJSON {
+            name: "Err: Contact not found".to_owned(),
+            email: "error@localhost".to_owned(),
+            profile_img: None,
+            color: "grey".to_owned(),
+        },
+    );
+    for author_id in chat_author_ids {
+        let contact = Contact::get_by_id(context, author_id).await;
+        if let Ok(c) = contact {
+            let profile_img_path: String;
+            if let Some(path) = c.get_profile_image(context).await {
+                profile_img_path = path
+                    .file_name()
+                    .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                    .to_str()
+                    .unwrap()
+                    .to_owned();
+                // push referenced blobs (avatars)
+                blobs.push(profile_img_path.clone());
+            } else {
+                profile_img_path = "".to_owned();
+            }
+            chat_authors.insert(
+                author_id,
+                ContactJSON {
+                    name: c.get_display_name().to_owned(),
+                    email: c.get_addr().to_owned(),
+                    profile_img: match profile_img_path != "" {
+                        true => Some(profile_img_path),
+                        false => None,
+                    },
+                    color: format!("{:#}", c.get_color()), // TODO
+                },
+            );
+        }
+    }
+
+    // Load information about the chat
+    let chat: Chat = Chat::load_from_db(context, chat_id).await.unwrap();
+    let chat_avatar = match chat.get_profile_image(context).await {
+        Some(img) => {
+            let path = img
+                .file_name()
+                .unwrap_or_else(|| std::ffi::OsStr::new(""))
+                .to_str()
+                .unwrap()
+                .to_owned();
+            blobs.push(path.clone());
+            Some(format!("blobs/{}", path))
+        }
+        None => None,
+    };
+
+    let chat_json = ChatJSON {
+        chat_json_version: 1,
+        export_timestamp: time(),
+        name: chat.get_name().to_owned(),
+        color: format!("{:#}", chat.get_color(&context).await),
+        profile_img: chat_avatar,
+        contacts: chat_authors,
+        referenced_external_messages,
+        messages: message_json,
+        locations: crate::location::get_range(&context, chat_id, 0, 0, crate::dc_tools::time())
+            .await,
+    };
+
+    blobs.sort();
+    blobs.dedup();
+    ExportChatResult {
+        chat_json: serde_json::to_string(&chat_json).unwrap(),
+        message_ids,
+        referenced_blobs: blobs,
+    }
+}

src/format_flowed.rs

@@ -146,6 +146,17 @@ mod tests {
 
         let text = "> Not a quote";
         assert_eq!(format_flowed(text), " > Not a quote");
+
+        // Test space stuffing of wrapped lines
+        let text = "> This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    >                               \n\
+                    > To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "\x20> This is the Autocrypt Setup Message used to transfer your key between \r\n\
+                        clients.\r\n\
+                        \x20>\r\n\
+                        \x20> To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
     }
 
     #[test]

src/headerdef.rs

@@ -11,16 +11,26 @@ pub enum HeaderDef {
     To,
     Cc,
     Disposition,
+
+    /// Used in the "Body Part Header" of MDNs as of RFC 8098.
+    /// Indicates the Message-ID of the message for which the MDN is being issued.
     OriginalMessageId,
 
     /// Delta Chat extension for message IDs in combined MDNs
     AdditionalMessageIds,
 
+    /// Outlook-SMTP-server replace the `Message-ID:`-header
+    /// and write the original ID to `X-Microsoft-Original-Message-ID`.
+    /// To sort things correctly and to not show outgoing messages twice,
+    /// we need to check that header as well.
+    XMicrosoftOriginalMessageId,
+
     ListId,
     References,
     InReplyTo,
     Precedence,
     ContentType,
+    ContentId,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -42,7 +52,9 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    Sender,
     EphemeralTimer,
+    Received,
     _TestHeader,
 }
 

src/html.rs

@@ -0,0 +1,553 @@
+///! # Get message as HTML.
+///!
+///! Use `Message.has_html()` to check if the UI shall render a
+///! corresponding button and `MsgId.get_html()` to get the full message.
+///!
+///! Even when the original mime-message is not HTML,
+///! `MsgId.get_html()` will return HTML -
+///! this allows nice quoting, handling linebreaks properly etc.
+use futures::future::FutureExt;
+use std::future::Future;
+use std::pin::Pin;
+
+use anyhow::Result;
+use lettre_email::mime::{self, Mime};
+
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::message::{Message, MsgId};
+use crate::mimeparser::parse_message_id;
+use crate::param::Param::SendHtml;
+use crate::plaintext::PlainText;
+use crate::{context::Context, message};
+use lettre_email::PartBuilder;
+use mailparse::ParsedContentType;
+
+impl Message {
+    /// Check if the message can be retrieved as HTML.
+    /// Typically, this is the case, when the mime structure of a Message is modified,
+    /// meaning that some text is cut or the original message
+    /// is in HTML and `simplify()` may hide some maybe important information.
+    /// The corresponding ffi-function is `dc_msg_has_html()`.
+    /// To get the HTML-code of the message, use `MsgId.get_html()`.
+    pub fn has_html(&self) -> bool {
+        self.mime_modified
+    }
+
+    /// Set HTML-part part of a message that is about to be sent.
+    /// The HTML-part is written to the database before sending and
+    /// used as the `text/html` part in the MIME-structure.
+    ///
+    /// Received HTML parts are handled differently,
+    /// they are saved together with the whole MIME-structure
+    /// in `mime_headers` and the HTML-part is extracted using `MsgId::get_html()`.
+    /// (To underline this asynchronicity, we are using the wording "SendHtml")
+    pub fn set_html(&mut self, html: Option<String>) {
+        if let Some(html) = html {
+            self.param.set(SendHtml, html);
+            self.mime_modified = true;
+        } else {
+            self.param.remove(SendHtml);
+            self.mime_modified = false;
+        }
+    }
+}
+
+/// Type defining a rough mime-type.
+/// This is mainly useful on iterating
+/// to decide whether a mime-part has subtypes.
+enum MimeMultipartType {
+    Multiple,
+    Single,
+    Message,
+}
+
+/// Function takes a content type from a ParsedMail structure
+/// and checks and returns the rough mime-type.
+async fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
+    let mimetype = ctype.mimetype.to_lowercase();
+    if mimetype.starts_with("multipart") && ctype.params.get("boundary").is_some() {
+        MimeMultipartType::Multiple
+    } else if mimetype == "message/rfc822" {
+        MimeMultipartType::Message
+    } else {
+        MimeMultipartType::Single
+    }
+}
+
+/// HtmlMsgParser converts a mime-message to HTML.
+#[derive(Debug)]
+struct HtmlMsgParser {
+    pub html: String,
+    pub plain: Option<PlainText>,
+}
+
+impl HtmlMsgParser {
+    /// Function takes a raw mime-message string,
+    /// searches for the main-text part
+    /// and returns that as parser.html
+    pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
+        let mut parser = HtmlMsgParser {
+            html: "".to_string(),
+            plain: None,
+        };
+
+        let parsedmail = mailparse::parse_mail(rawmime)?;
+
+        parser.collect_texts_recursive(context, &parsedmail).await?;
+
+        if parser.html.is_empty() {
+            if let Some(plain) = &parser.plain {
+                parser.html = plain.to_html().await;
+            }
+        } else {
+            parser.cid_to_data_recursive(context, &parsedmail).await?;
+        }
+
+        Ok(parser)
+    }
+
+    /// Function iterates over all mime-parts
+    /// and searches for text/plain and text/html parts and saves the
+    /// first one found.
+    /// in the corresponding structure fields.
+    ///
+    /// Usually, there is at most one plain-text and one HTML-text part,
+    /// multiple plain-text parts might be used for mailinglist-footers,
+    /// therefore we use the first one.
+    fn collect_texts_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.collect_texts_recursive(context, cur_data).await?
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.collect_texts_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype == mime::TEXT_HTML {
+                        if self.html.is_empty() {
+                            if let Ok(decoded_data) = mail.get_body() {
+                                self.html = decoded_data;
+                            }
+                        }
+                    } else if mimetype == mime::TEXT_PLAIN && self.plain.is_none() {
+                        if let Ok(decoded_data) = mail.get_body() {
+                            self.plain = Some(PlainText {
+                                text: decoded_data,
+                                flowed: if let Some(format) = mail.ctype.params.get("format") {
+                                    format.as_str().to_ascii_lowercase() == "flowed"
+                                } else {
+                                    false
+                                },
+                                delsp: if let Some(delsp) = mail.ctype.params.get("delsp") {
+                                    delsp.as_str().to_ascii_lowercase() == "yes"
+                                } else {
+                                    false
+                                },
+                            });
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+
+    /// Replace cid:-protocol by the data:-protocol where appropriate.
+    /// This allows the final html-file to be self-contained.
+    fn cid_to_data_recursive<'a>(
+        &'a mut self,
+        context: &'a Context,
+        mail: &'a mailparse::ParsedMail<'a>,
+    ) -> Pin<Box<dyn Future<Output = Result<()>> + 'a + Send>> {
+        // Boxed future to deal with recursion
+        async move {
+            match get_mime_multipart_type(&mail.ctype).await {
+                MimeMultipartType::Multiple => {
+                    for cur_data in mail.subparts.iter() {
+                        self.cid_to_data_recursive(context, cur_data).await?;
+                    }
+                    Ok(())
+                }
+                MimeMultipartType::Message => {
+                    let raw = mail.get_body_raw()?;
+                    if raw.is_empty() {
+                        return Ok(());
+                    }
+                    let mail = mailparse::parse_mail(&raw).unwrap();
+                    self.cid_to_data_recursive(context, &mail).await
+                }
+                MimeMultipartType::Single => {
+                    let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
+                    if mimetype.type_() == mime::IMAGE {
+                        if let Some(cid) = mail.headers.get_header_value(HeaderDef::ContentId) {
+                            if let Ok(cid) = parse_message_id(&cid) {
+                                if let Ok(replacement) = mimepart_to_data_url(mail).await {
+                                    let re_string = format!(
+                                        "(<img[^>]*src[^>]*=[^>]*)(cid:{})([^>]*>)",
+                                        regex::escape(&cid)
+                                    );
+                                    match regex::Regex::new(&re_string) {
+                                        Ok(re) => {
+                                            self.html = re
+                                                .replace_all(
+                                                    &*self.html,
+                                                    format!("${{1}}{}${{3}}", replacement).as_str(),
+                                                )
+                                                .as_ref()
+                                                .to_string()
+                                        }
+                                        Err(e) => warn!(
+                                            context,
+                                            "Cannot create regex for cid: {} throws {}",
+                                            re_string,
+                                            e
+                                        ),
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    Ok(())
+                }
+            }
+        }
+        .boxed()
+    }
+}
+
+/// Convert a mime part to a data: url as defined in [RFC 2397](https://tools.ietf.org/html/rfc2397).
+async fn mimepart_to_data_url(mail: &mailparse::ParsedMail<'_>) -> Result<String> {
+    let data = mail.get_body_raw()?;
+    let data = base64::encode(&data);
+    Ok(format!("data:{};base64,{}", mail.ctype.mimetype, data))
+}
+
+impl MsgId {
+    /// Get HTML from a message-id.
+    /// This requires `mime_headers` field to be set for the message;
+    /// this is the case at least when `Message.has_html()` returns true
+    /// (we do not save raw mime unconditionally in the database to save space).
+    /// The corresponding ffi-function is `dc_get_msg_html()`.
+    pub async fn get_html(self, context: &Context) -> Result<Option<String>> {
+        let rawmime = message::get_mime_headers(context, self).await?;
+
+        if !rawmime.is_empty() {
+            match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
+                Err(err) => {
+                    warn!(context, "get_html: parser error: {}", err);
+                    Ok(None)
+                }
+                Ok(parser) => Ok(Some(parser.html)),
+            }
+        } else {
+            warn!(context, "get_html: no mime for {}", self);
+            Ok(None)
+        }
+    }
+}
+
+/// Wraps HTML text into a new text/html mimepart structure.
+///
+/// Used on forwarding messages to avoid leaking the original mime structure
+/// and also to avoid sending too much, maybe large data.
+pub async fn new_html_mimepart(html: String) -> PartBuilder {
+    PartBuilder::new()
+        .content_type(&"text/html; charset=utf-8".parse::<mime::Mime>().unwrap())
+        .body(html)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::chat;
+    use crate::chat::forward_msgs;
+    use crate::config::Config;
+    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message::MessengerMessage;
+    use crate::test_utils::TestContext;
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_unspecified() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This message does not have Content-Type nor Subject.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_iso88591() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_plain_flowed() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.plain.unwrap().flowed);
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+This line ends with a space and will be merged with the next one due to format=flowed.<br/>
+<br/>
+This line does not end with a space<br/>
+and will be wrapped as usual.<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html,
+            r##"<!DOCTYPE html>
+<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /></head><body>
+mime-modified should not be set set as there is no html and no special stuff;<br/>
+although not being a delta-message.<br/>
+test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x27; :)<br/>
+<br/>
+<br/>
+</body></html>
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+
+        // on windows, `\r\n` linends are returned from mimeparser,
+        // however, rust multiline-strings use just `\n`;
+        // therefore, we just remove `\r` before comparison.
+        assert_eq!(
+            parser.html.replace("\r", ""),
+            r##"
+<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>mime-modified <b>set</b>; simplify is always regarded as lossy.</p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_alt_plain_html() {
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert_eq!(
+            parser.html.replace("\r", ""), // see comment in test_htmlparse_html()
+            r##"<html>
+  <p>
+    this is <b>html</b>
+  </p>
+</html>
+
+"##
+        );
+    }
+
+    #[async_std::test]
+    async fn test_htmlparse_apple_cid_jpg() {
+        // load raw mime html-data with related image-part (cid:)
+        // and make sure, Content-Id has angle-brackets that are removed correctly.
+        let t = TestContext::new().await;
+        let raw = include_bytes!("../test-data/message/apple_cid_jpg.eml");
+        let test = String::from_utf8_lossy(raw);
+        assert!(test.contains("Content-Id: <8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box>"));
+        assert!(test.contains("cid:8AE052EF-BC90-486F-BB78-58D3590308EC@fritz.box"));
+        assert!(test.find("data:").is_none());
+
+        // parsing converts cid: to data:
+        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        assert!(parser.html.contains("<html>"));
+        assert!(!parser.html.contains("Content-Id:"));
+        assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ"));
+        assert!(!parser.html.contains("cid:"));
+    }
+
+    #[async_std::test]
+    async fn test_get_html_empty() {
+        let t = TestContext::new().await;
+        let msg_id = MsgId::new(100);
+        assert!(msg_id.get_html(&t).await.unwrap().is_none())
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding() {
+        // alice receives a non-delta html-message
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("2")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::No);
+        assert!(!msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap().unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // alice: create chat with bob and forward received html-message there
+        let chat = alice.create_chat_with_contact("", "bob@example.net").await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap().unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+
+        // bob: check that bob also got the html-part of the forwarded message
+        let bob = TestContext::new_bob().await;
+        let chat = bob.create_chat_with_contact("", "alice@example.com").await;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat.get_id()).await;
+        assert_ne!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&bob).await.unwrap().unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_html_forwarding_encrypted() {
+        // Alice receives a non-delta html-message
+        // (`ShowEmails=1` lets Alice actually receive non-delta messages for known contacts,
+        // the contact is marked as known by creating a chat using `chat_with_contact()`)
+        let alice = TestContext::new_alice().await;
+        alice.set_config(Config::ShowEmails, Some("1")).await.ok();
+        let chat = alice
+            .create_chat_with_contact("", "sender@testrun.org")
+            .await;
+        let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
+        dc_receive_imf(&alice, raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+
+        // forward the message to saved-messages,
+        // this will encrypt the message as new_alice() has set up keys
+        let chat = alice.get_self_chat().await;
+        forward_msgs(&alice, &[msg.get_id()], chat.get_id())
+            .await
+            .unwrap();
+        let msg = alice.pop_sent_msg().await;
+
+        // receive the message on another device
+        let alice = TestContext::new_alice().await;
+        assert_eq!(alice.get_config_int(Config::ShowEmails).await.unwrap(), 0); // set to "1" above, make sure it is another db
+        alice.recv_msg(&msg).await;
+        let chat = alice.get_self_chat().await;
+        let msg = alice.get_last_msg_in(chat.get_id()).await;
+        assert_eq!(msg.get_from_id(), DC_CONTACT_ID_SELF);
+        assert_eq!(msg.is_dc_message, MessengerMessage::Yes);
+        assert!(msg.get_showpadlock());
+        assert!(msg.is_forwarded());
+        assert!(msg.get_text().unwrap().contains("this is plain"));
+        assert!(msg.has_html());
+        let html = msg.get_id().get_html(&alice).await.unwrap().unwrap();
+        assert!(html.contains("this is <b>html</b>"));
+    }
+
+    #[async_std::test]
+    async fn test_set_html() {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // alice sends a message with html-part to bob
+        let chat_id = alice.create_chat(&bob).await.id;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("plain text".to_string()));
+        msg.set_html(Some("<b>html</b> text".to_string()));
+        assert!(msg.mime_modified);
+        chat::send_msg(&alice, chat_id, &mut msg).await.unwrap();
+
+        // check the message is written correctly to alice's db
+        let msg = alice.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&alice).await.unwrap().unwrap();
+        assert!(html.contains("<b>html</b> text"));
+
+        // let bob receive the message
+        let chat_id = bob.create_chat(&alice).await.id;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+        let msg = bob.get_last_msg_in(chat_id).await;
+        assert_eq!(msg.get_text(), Some("plain text".to_string()));
+        assert!(!msg.is_forwarded());
+        assert!(msg.mime_modified);
+        let html = msg.get_id().get_html(&bob).await.unwrap().unwrap();
+        assert!(html.contains("<b>html</b> text"));
+    }
+}