just print events

just print events instead of using the info!() warn!() error!() macros
that result in an event beeing emmited again.

.circleci/config.yml

@@ -7,13 +7,116 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
             
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
 jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
   build_doxygen:
     executor: doxygen
     steps:
       - checkout
-      - run: bash scripts/run-doxygen.sh
+      - run: bash ci_scripts/run-doxygen.sh
       - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
@@ -27,7 +130,7 @@ jobs:
       - checkout
       # the following commands on success produces
       # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash scripts/remote_python_packaging.sh
+      - run: bash ci_scripts/remote_python_packaging.sh 
       - persist_to_workspace:
           root: workspace
           paths:
@@ -35,34 +138,99 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
   upload_docs_wheels:
     machine: true
     steps:
       - checkout
       - attach_workspace:
           at: workspace
+      - run: pyenv versions
+      - run: pyenv global 3.5.2
       - run: ls -laR workspace
-      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy 
+
 
 workflows:
   version: 2.1
 
   test:
     jobs:
-      - remote_python_packaging:
+      # - cargo_fetch
+
+      - remote_tests_rust:
           filters:
-            branches:
-              only: master
+            tags:
+              only: /.*/
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+             - remote_tests_rust 
+          filters:
+            tags:
+              only: /.*/
 
       - upload_docs_wheels:
           requires:
              - remote_python_packaging
              - build_doxygen
           filters:
-            branches:
-              only: master
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
 
       - build_doxygen:
           filters:
-            branches:
-              only: master
+            tags:
+              only: /.*/
+
+      # Linux Desktop 64bit
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -1,127 +0,0 @@
-name: Rust CI
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - staging
-      - trying
-
-jobs:
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.50.0
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: 1.50.0
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
-
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
-
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      matrix:
-        # macOS disabled due to random failures related to caching
-        #os: [ubuntu-latest, windows-latest, macOS-latest]
-        os: [ubuntu-latest, windows-latest]
-        rust: [1.50.0]
-        experimental: [false]
-#        include:
-#          - os: ubuntu-latest
-#            rust: nightly
-#            experimental: true
-#          - os: windows-latest
-#            rust: nightly
-#            experimental: true
-#          - os: macOS-latest
-#            rust: nightly
-#            experimental: true
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
-      with:
-        command: check
-        args: --all --bins --examples --tests --features repl
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all

.github/workflows/code-quality.yml

@@ -0,0 +1,48 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-19
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --workspace --examples --tests --all-features
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-19
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2020-03-19
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

.github/workflows/remote_tests.yml

@@ -1,18 +0,0 @@
-name: Remote tests
-on: [push]
-jobs:
-  remote_tests_python:
-    name: Remote Python tests
-    runs-on: ubuntu-latest
-    env:
-      DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-    steps:
-      - uses: actions/checkout@v2
-      - run: mkdir -m 700 -p ~/.ssh
-      - run: touch ~/.ssh/id_ed25519
-      - run: chmod 600 ~/.ssh/id_ed25519
-      - run: 'echo "$SSH_KEY" | base64 -d > ~/.ssh/id_ed25519'
-        shell: bash
-        env:
-          SSH_KEY: ${{ secrets.SSH_KEY }}
-      - run: scripts/remote_tests_python.sh "deltachat-core/python/${{ github.ref }}/${{ github.run_number }}"

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -26,5 +25,3 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
-
-coverage/

ASYNC-API-TODO.txt

@@ -0,0 +1,54 @@
+
+Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
+
+- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
+
+APIs:
+    dc_context_new                 # opens the database
+    dc_open                             # FFI only
+      -> drop it and move parameters to dc_context_new()
+
+    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
+    dc_imex NEVER goes through the job system
+    dc_imex import_backup needs to ensure dc_stop_jobs()
+
+    dc_start_io                # start smtp/imap and job handling subsystems
+    dc_stop_io                 # stop smtp/imap and job handling subsystems
+    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
+
+    dc_close                            # FFI only
+      -> can be dropped
+    dc_context_unref
+
+    for ios share-extension:
+    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
+        0: message was sent
+        1: message failed to go out, is queued as a job to be retried later
+        2: message permanently failed?
+
+EVENT handling:
+    start a callback thread and call get_next_event() which is BLOCKING
+    it's fine to start this callback thread later, it will see all events.
+    Note that the core infinitely fills the internal queue if you never drain it.
+
+    FFI-get_next_event() returns NULL if the context is unrefed already?
+
+   sidenote: how python's callback thread does it currently:
+         CB-thread runs this while loop:
+         while not QUITFLAG:
+             ev = context.get_next_event( )
+             ...
+         So in order to shutdown properly one has to set QUITFLAG
+         before calling dc_stop_jobs() and dc_context_unref
+
+    event API:
+        get_data1_int
+        get_data2_int
+        get_data3_str
+
+
+- userdata likely only used for the callbacks, likely can be dropped, needs verification
+
+
+- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
+

CHANGELOG.md

@@ -1,580 +1,5 @@
 # Changelog 
 
-## 1.55.0
-
-- fix panic when receiving some HTML messages #2434
-
-- fix downloading some messages multiple times #2430
-
-- fix formatting of read receipt texts #2431
-
-- simplify SQL error handling #2415
-
-- explicit rust API for creating chats with blocked status #2282
-
-- debloat the binary by using less AsRef arguments #2425
-
-
-## 1.54.0
-
-- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
-
-- global search performance improvement #2364 #2365 #2366
-
-- improve SQLite performance with `PRAGMA synchronous=normal` #2382
-
-- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
-
-- python: list `requests` as a requirement #2390
-
-- fix creation of many delete jobs when being offline #2372
-
-- synchronize status between devices #2386
-
-- deaddrop (contact requests) chat improvements #2373
-
-- add "Forwarded:" to notification and chatlist summaries #2310
-
-- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
-
-- improve tests #2360 #2362 #2370 #2377 #2387
-
-- cleanup #2359 #2361 #2374 #2376 #2379 #2388
-
-
-## 1.53.0
-
-- fix sqlx performance regression #2355 2356
-
-- add a `ci_scripts/coverage.sh` #2333 #2334
-
-- refactorings and tests #2348 #2349 #2350
-
-- improve python bindings #2332 #2326
-
-
-## 1.52.0
-
-- database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340
-
-- add alias support: UIs should check for `dc_msg_get_override_sender_name()`
-  also in single-chats now and display divergent names and avatars #2297
-
-- parse blockquote-tags for better quote detection #2313
-
-- ignore unknown classical emails from spam folder #2311
-
-- support "Mixed Up” encryption repairing #2321
-
-- fix single chat search #2344
-
-- fix nightly clippy and rustc errors #2341
-
-- update dependencies #2350
-
-- improve ci #2342
-
-- improve python bindings #2332 #2326
-
-
-## 1.51.0
-
-- breaking change: You have to call `dc_stop_io()`/`dc_start_io()`
-  before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`:
-  fix race condition and db corruption
-  when a message was received during backup #2253
-
-- save subject for messages: new api `dc_msg_get_subject()`,
-  when quoting, use the subject of the quoted message as the new subject,
-  instead of the last subject in the chat #2274 #2283
-
-- new apis to get full or html message,
-  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151 #2264 #2279
-
-- new chat type and apis for the new mailing list support,
-  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
-  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
-  #2241 #2243 #2258 #2259 #2261 #2267 #2270 #2272 #2290
-
-- new api `dc_decide_on_contact_request()`,
-  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
-
-- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
-
-- new api `dc_get_chat_encrinfo()` #2186
-
-- new api `dc_contact_get_status()`, returning the recent footer #2218 #2307
-
-- improve contact name update rules,
-  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
-
-- new api for bots: `dc_msg_set_html()` #2153
-
-- new api for bots: `dc_msg_set_override_sender_name()` #2231
-
-- api removed: `dc_is_io_running()` #2139
-
-- api removed: `dc_contact_get_first_name()` #2165 #2171
-
-- improve compatibility with providers changing the Message-ID
-  (as Outlook.com) #2250 #2265
-
-- correctly show emails that were sent to an alias and then bounced 
-
-- implement Consistent Color Generation (XEP-0392),
-  that results in contact colors be be changed #2228 #2229 #2239
-
-- fetch recent existing messages
-  and create corresponding chats after configure #2106
-
-- improve e-mail compatibility
-  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
-
-- better support videochat-services not supporting random rooms #2191
-
-- export backups as .tar files #2023
-
-- scale avatars based on media_quality, fix avatar rotation #2063
-
-- compare ephemeral timer to parent message to deal with reordering better #2100
-
-- better ephemeral system messages #2183
-
-- read quotes out of html messages #2104
-
-- prepend subject to messages with attachments, if needed #2111
-
-- run housekeeping at least once a day #2114
-
-- resolve MX domain only once per OAuth2 provider #2122
-
-- configure provider based on MX record #2123 #2134
-
-- make transient bad destination address error permanent
-  after n tries #2126 #2202
-
-- enable strict TLS for known providers by default #2121
-
-- improve and harden secure join #2154 #2161 #2251
-
-- update `dc_get_info()` to return more information #2156
-
-- prefer In-Reply-To/References
-  over group-id stored in Message-ID #2164 #2172 #2173
-
-- apply gossiped encryption preference to new peerstates #2174
-
-- fix: do not return quoted messages from the trash chat #2221
-
-- fix: allow emojis for location markers #2177
-
-- fix encoding of Chat-Group-Name-Changed messages that could even lead to
-  messages not being delivered #2141
-
-- fix error when no temporary directory is available #1929
-
-- fix marking read receipts as seen #2117
-
-- fix read-notification for mixed-case addresses #2103
-
-- fix decoding of attachment filenames #2080 #2094 #2102
-
-- fix downloading ranges of message #2061
-
-- fix parsing quoted encoded words in From: header #2193 #2204
-
-- fix import/export race condition #2250
-
-- fix: exclude muted chats from notified-list #2269 #2275
-
-- fix: update uid_next if the server rewind it #2288
-
-- fix: return error on fingerprint mismatch on qr-scan #2295
-
-- fix ci #2217 #2226 #2244 #2245 #2249 #2277 #2286
-
-- try harder on backup opening #2148
-
-- trash messages more thoroughly #2273
-
-- nicer logging #2284
-
-- add CMakeLists.txt #2260
-
-- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
-
-- improve python bindings #2113 #2115 #2133 #2214
-
-- improve documentation #2143 #2160 #2175 #2146
-
-- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
-  #2200 #2230 #2262 #2203
-
-- update provider-database #2299
-
-
-## 1.50.0
-
-- do not fetch emails in between inbox_watch disabled and enabled again #2087
-
-- fix: do not fetch from INBOX if inbox_watch is disabled #2085
-
-- fix: do not use STARTTLS when PLAIN connection is requested
-  and do not allow downgrade if STARTTLS is not available #2071
-
-
-## 1.49.0
-
-- add timestamps to image and video filenames #2068
-
-- forbid quoting messages from another context #2069
-
-- fix: preserve quotes in messages with attachments #2070
-
-
-## 1.48.0
-
-- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
-  #2035 #2042
-
-- skip fetch existing messages/contacts if config-option `bot` set #2017
-
-- always log why a message is sorted to trash #2045
-
-- display a quote if top posting is detected #2047
-
-- add ephemeral task cancellation to `dc_stop_io()`;
-  before, there was no way to quickly terminate pending ephemeral tasks #2051
-
-- when saved-messages chat is deleted,
-  a device-message about recreation is added #2050
-
-- use `max_smtp_rcpt_to` from provider-db,
-  sending messages to many recipients in configurable chunks #2056
-
-- fix handling of empty autoconfigure files #2027
-
-- fix adding saved messages to wrong chats on multi-device #2034 #2039
-
-- fix hang on android4.4 and other systems
-  by adding a workaround to executer-blocking-handling bug #2040
-
-- fix secret key export/import roundtrip #2048
-
-- fix mistakenly unarchived chats #2057
-
-- fix outdated-reminder test that fails only 7 days a year,
-  including halloween :) #2059
-
-- improve python bindings #2021 #2036 #2038
-
-- update provider-database #2037
-
-
-## 1.47.0
-
-- breaking change: `dc_update_device_chats()` removed;
-  this is now done automatically during configure
-  unless the new config-option `bot` is set #1957
-
-- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
-  and remove `dc_marknoticed_all_chats()` #1942 #1981
-
-- breaking change: remove unused starring options #1965
-
-- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
-  `dc_chat_is_protected()`; also single-chats may be protected now, this may
-  happen over the wire even if the UI do not offer an option for that #1968
-
-- breaking change: split quotes off message text,
-  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
-
-- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
-  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
-
-- require quorum to enable encryption #1946
-
-- speed up and clean up account creation #1912 #1927 #1960 #1961
-
-- configure now collects recent contacts and fetches last messages
-  unless disabled by `fetch_existing` config-option #1913 #2003
-  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
-
-- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
-  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
-
-- add `dc_set_chat_protection()`; the `protect` parameter in
-  `dc_create_group_chat()` will be removed in an upcoming release;
-  up to then, UIs using the "verified group" paradigm
-  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
-
-- remove unneeded `DC_STR_COUNT` #1991
-
-- mark all failed messages as failed when receiving an NDN #1993
-
-- check some easy cases for bad system clock and outdated app #1901
-
-- fix import temporary directory usage #1929
-
-- fix forcing encryption for reset peers #1998
-
-- fix: do not allow to save drafts in non-writeable chats #1997
-
-- fix: do not show HTML if there is no content and there is an attachment #1988
-
-- fix recovering offline/lost connections, fixes background receive bug #1983
-
-- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
-
-- fix whitespace for summaries #1938
-
-- fix: improve sentbox name guessing #1941
-
-- fix: avoid manual poll impl for accounts events #1944
-
-- fix encoding newlines in param as a preparation for storing quotes #1945
-
-- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
-
-- fix ci #1928 #1931 #1932 #1933 #1934 #1943
-
-- update provider-database #1940 #2005 #2006
-
-- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
-
-
-## 1.46.0
-
-- breaking change: `dc_configure()` report errors in
-  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
-  #1886 #1905
-
-- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
-  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
-
-- parse multiple servers in Mozilla autoconfig #1860
-
-- try multiple servers for each protocol #1871
-
-- do IMAP and SMTP configuration in parallel #1891
-
-- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
-
-- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
-
-- do not reset peerstate on encrypted messages,
-  ignore reordered autocrypt headers #1885 #1890
-
-- always sort message replies after parent message #1852
-
-- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
-
-- improve mimetype guessing for PDF and many other formats #1857 #1861
-
-- improve accepting invalid html #1851
-
-- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
-
-- tweak HELO command #1908
-
-- make `dc_accounts_get_all()` return accounts sorted #1909
-
-- fix KML coordinates precision used for location streaming #1872
-
-- fix cancelling import/export #1855
-
-
-## 1.45.0
-
-- add `dc_accounts_t` account manager object and related api functions #1784
-
-- add capability to import backups as .tar files,
-  which will become the default in a subsequent release #1749
-
-- try various server domains on configuration #1780 #1838
-
-- recognize .tgs files as stickers #1826
-
-- remove X-Mailer debug header #1819
-
-- improve guessing message types from extension #1818
-
-- fix showing unprotected subjects in encrypted messages #1822
-
-- fix threading in interaction with non-delta-clients #1843
-
-- fix handling if encryption degrades #1829
-
-- fix webrtc-servers names set by the user #1831
-
-- update provider database #1828
-
-- update async-imap to fix Oauth2 #1837
-
-- optimize jpeg assets with trimage #1840
-
-- add tests and documentations #1809 #1820
-
-
-## 1.44.0
-
-- fix peerstate issues #1800 #1805
-
-- fix a crash related to muted chats #1803
-
-- fix incorrect dimensions sometimes reported for images #1806
-
-- fixed `dc_chat_get_remaining_mute_duration` function #1807
-
-- handle empty tags (e.g. `<br/>`) in HTML mails #1810
-
-- always translate the message about disappearing messages timer change #1813
-
-- improve footer detection in plain text email #1812
-
-- update device chat icon to fix warnings in iOS logs #1802
-
-- fix deletion of multiple messages #1795
-
-
-## 1.43.0
-
-- improve using own jitsi-servers #1785
-
-- fix smtp-timeout tweaks for larger mails #1797
-
-- more bug fixes and updates #1794 #1792 #1789 #1787
-
-
-## 1.42.0
-
-- new qr-code type `DC_QR_WEBRTC` #1779
-
-- new `dc_chatlist_get_summary2()` api #1771
-
-- tweak smtp-timeout for larger mails #1782
-
-- optimize read-receipts #1765
-
-- Allow http scheme for DCACCOUNT URLs #1770
-
-- improve tests #1769
-
-- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
-
-
-## 1.41.0
-
-- new apis to initiate video chats #1718 #1735
-
-- new apis `dc_msg_get_ephemeral_timer()`
-  and `dc_msg_get_ephemeral_timestamp()`
-
-- new api `dc_chatlist_get_summary2()` #1771
-
-- improve IMAP handling #1703 #1704
-
-- improve ephemeral messages #1696 #1705
-
-- mark location-messages as auto-generated #1715
-
-- multi-device avatar-sync #1716 #1717
-
-- improve python bindings #1732 #1733 #1738 #1769
-
-- Allow http scheme for DCACCOUNT urls #1770
-
-- more fixes #1702 #1706 #1707 #1710 #1719 #1721
-  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
-
-- refactorings #1712 #1714 #1757
-
-- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
-
-
-## 1.40.0
-
-- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
-
-- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
-
-- improve idle #1690 #1688
-
-- fix message processing issues by sequential processing #1694
-
-- refactorings #1670 #1673
-
-
-## 1.39.0
-
-- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
-
-- fix potential panics, update dependencies #1650 #1655
-
-
-## 1.38.0
-
-- fix sorting, esp. for multi-device
-
-
-## 1.37.0
-
-- improve ndn heuristics #1630
-
-- get oauth2 authorizer from provider-db #1641
-
-- removed linebreaks and spaces from generated qr-code #1631
-
-- more fixes #1633 #1635 #1636 #1637
-
-
-## 1.36.0
-
-- parse ndn (network delivery notification) reports
-  and report failed messages as such #1552 #1622 #1630
-
-- add oauth2 support for gsuite domains #1626
-
-- read image orientation from exif before recoding #1619
-
-- improve logging #1593 #1598
-
-- improve python and bot bindings #1583 #1609
-
-- improve imap logout #1595
-
-- fix sorting #1600 #1604
-
-- fix qr code generation #1631
-
-- update rustcrypto releases #1603
-
-- refactorings #1617
-
-
-## 1.35.0
-
-- enable strict-tls from a new provider-db setting #1587
-
-- new subject 'Message from USER' for one-to-one chats #1395
-
-- recode images #1563
-
-- improve reconnect handling #1549 #1580
-
-- improve importing addresses #1544
-
-- improve configure and folder detection #1539 #1548
-
-- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
-
-- fix ad-hoc groups #1566
-
-- preventions against being marked as spam #1575
-
-- refactorings #1542 #1569
-
-
 ## 1.34.0
 
 - new api for io, thread and event handling #1356,
@@ -1088,3 +513,4 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

CMakeLists.txt

@@ -1,38 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat)
-
-find_program(CARGO cargo)
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-include(GNUInstallDirs)
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,89 +1,78 @@
 [package]
 name = "deltachat"
-version = "1.55.0"
+version = "1.34.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
-[profile.dev]
-debug = 0
-
 [profile.release]
-lto = true
+# lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.40"
-async-imap = "0.5.0"
-async-native-tls = { version = "0.3.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
-async-std-resolver = "0.20.2"
-async-std = { version = "~1.9.0", features = ["unstable"] }
-async-tar = "0.3.0"
-async-trait = "0.1.50"
-backtrace = "0.3.59"
-base64 = "0.13"
-bitflags = "1.1.0"
-byteorder = "1.3.1"
-charset = "0.1"
-chrono = "0.4.6"
-dirs = { version = "3.0.2", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.1"
-futures = "0.3.14"
-hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.3.0"
-itertools = "0.10.0"
-kamadak-exif = "0.5"
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2.51"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13.4"
-native-tls = "0.2.3"
-num_cpus = "1.13.0"
+pgp = { version = "0.5.1", default-features = false } 
+hex = "0.4.0"
+sha2 = "0.8.0"
+rand = "0.7.0"
+smallvec = "1.0.0"
+surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-once_cell = "1.4.1"
+async-smtp = { version = "0.3" }
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+async-imap = "0.3.1"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.0", features = ["unstable"] }
+base64 = "0.11"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7.0", default-features = false }
-pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.22.0"
-r2d2 = "0.8.9"
-r2d2_sqlite = "0.18.0"
-rand = "0.7.0"
-regex = "1.4.6"
-rusqlite = "0.25"
-rust-hsluv = "0.1.4"
-rustyline = { version = "8.0.0", optional = true }
-sanitize-filename = "0.3.0"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.5"
-sha2 = "0.9.4"
-smallvec = "1.0.0"
-stop-token = "0.2.0"
-strum = "0.20.0"
-strum_macros = "0.20.1"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+serde_json = "1.0"
+chrono = "0.4.6"
+indexmap = "1.3.0"
+lazy_static = "1.4.0"
+regex = "1.1.6"
+rusqlite = { version = "0.22", features = ["bundled"] }
+r2d2_sqlite = "0.15.0"
+r2d2 = "0.8.5"
+strum = "0.16.0"
+strum_macros = "0.16.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.8.0"
+image-meta = "0.1.0"
+quick-xml = "0.17.1"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.12.1"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
 thiserror = "1.0.14"
-toml = "0.5.6"
-url = "2.2.2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
+anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+
+pretty_env_logger = { version = "0.3.1", optional = true }
+log = {version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
+
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
-criterion = "0.3"
-futures-lite = "1.7.0"
-log = "0.4.11"
-pretty_assertions = "0.7.2"
-pretty_env_logger = "0.4.0"
-proptest = "1.0"
 tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.3.0"
+proptest = "0.9.4"
+async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+smol = "0.1.10"
 
 [workspace]
 members = [
@@ -102,21 +91,12 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
-[[bench]]
-name = "create_account"
-harness = false
-
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
+
+[patch.crates-io]
+smol = { git = "https://github.com/dignifiedquire/smol-1", branch = "isolate-nix" }

README.md

@@ -2,9 +2,7 @@
 
 > Deltachat-core written in Rust 
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
-[![Remote tests](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml)
-[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
 ## Installing Rust and Cargo
 
@@ -19,7 +17,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -97,8 +95,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
-SMTP tracing in addition to info messages.
+
 
 ### Expensive tests
 
@@ -113,6 +110,11 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
@@ -120,8 +122,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
-- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
+- [Go](https://github.com/hugot/go-deltachat/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

appveyor.yml

@@ -0,0 +1,19 @@
+environment:
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init -yv --default-toolchain nightly-2020-03-19
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+  - rustc -vV
+  - cargo -vV
+
+build: false
+
+test_script:
+  - cargo test --release --all
+
+cache:
+  - target
+  - C:\Users\appveyor\.cargo\registry

benches/contacts.rs

@@ -1,39 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/create_account.rs

@@ -1,26 +0,0 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::accounts::Accounts;
-use tempfile::tempdir;
-
-async fn create_accounts(n: u32) {
-    let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
-
-    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-
-    for expected_id in 2..n {
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, expected_id);
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,29 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -1,15 +1,11 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions),
-[CircleCI](https://app.circleci.com/) and an own build machine.
+Continuous Integration, run through CircleCI and an own build machine.
 
 ## Description of scripts 
 
-- `../.github/workflows` contains jobs run by GitHub Actions.
-
 - `../.circleci/config.yml` describing the build jobs that are run
-  by CircleCI.
+  by Circle-CI
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -30,8 +26,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -45,6 +41,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd scripts  # where all CI things are 
+    cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 

ci_scripts/ci_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install -U pip setuptools
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*
+
+popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/docker-coredeps/Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust
+# Install Rust nightly
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -1,9 +1,11 @@
 #!/bin/bash 
 
-export BRANCH=${CIRCLE_BRANCH:-master}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -28,15 +30,15 @@ set +x
 
 ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
-    shopt -s huponexit
     cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
     set -x
 
     # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL \
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 _HERE
 

ci_scripts/remote_tests_python.sh

@@ -1,9 +1,12 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,17 +20,16 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
-
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -41,5 +43,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash scripts/run-python-test.sh 
+    bash ci_scripts/run-python-test.sh 
 _HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -4,7 +4,6 @@
 # and tox/pytest. 
 
 set -e -x
-shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
 set -ex
-shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,7 +15,6 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
-export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
@@ -39,6 +38,7 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
+unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
 tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.55.0"
+version = "1.34.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,10 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.9.0"
-anyhow = "1.0.40"
+async-std = "1.6.0"
+anyhow = "1.0.28"
 thiserror = "1.0.14"
-rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -236,6 +236,12 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
+# This tag can be used to specify a number of word-keyword mappings (TCL only).
+# A mapping has the form "name=value". For example adding "class=itcl::class"
+# will allow you to use the command class in the itcl::class meaning.
+
+TCL_SUBST              =
+
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/src/dc_array.rs

@@ -1,56 +1,46 @@
-use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
-use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
-    MsgIds(Vec<MsgId>),
-    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub(crate) fn get_id(&self, index: usize) -> u32 {
+    pub fn new(capacity: usize) -> Self {
+        dc_array_t::Uint(Vec::with_capacity(capacity))
+    }
+
+    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
+    pub fn new_locations(capacity: usize) -> Self {
+        dc_array_t::Locations(Vec::with_capacity(capacity))
+    }
+
+    pub fn add_id(&mut self, item: u32) {
+        if let Self::Uint(array) = self {
+            array.push(item);
+        } else {
+            panic!("Attempt to add id to array of other type");
+        }
+    }
+
+    pub fn add_location(&mut self, location: Location) {
+        if let Self::Locations(array) = self {
+            array.push(location)
+        } else {
+            panic!("Attempt to add a location to array of other type");
+        }
+    }
+
+    pub fn get_id(&self, index: usize) -> u32 {
         match self {
-            Self::MsgIds(array) => array[index].to_u32(),
-            Self::Chat(array) => match array[index] {
-                ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
-                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
-            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index],
+            Self::Uint(array) => array[index] as u32,
         }
     }
 
-    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(array) => array.get(index).and_then(|item| match item {
-                ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
-                ChatItem::DayMarker { timestamp } => Some(*timestamp),
-            }),
-            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
-        match self {
-            Self::MsgIds(_) => None,
-            Self::Chat(_) => None,
-            Self::Locations(array) => array
-                .get(index)
-                .and_then(|location| location.marker.as_deref()),
-            Self::Uint(_) => None,
-        }
-    }
-
-    pub(crate) fn get_location(&self, index: usize) -> &Location {
+    pub fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -58,18 +48,55 @@ impl dc_array_t {
         }
     }
 
-    /// Returns the number of elements in the array.
-    pub(crate) fn len(&self) -> usize {
+    pub fn is_empty(&self) -> bool {
+        match self {
+            Self::Locations(array) => array.is_empty(),
+            Self::Uint(array) => array.is_empty(),
+        }
+    }
+
+    /// Returns the number of elements in the array.
+    pub fn len(&self) -> usize {
         match self {
-            Self::MsgIds(array) => array.len(),
-            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
-        (0..self.len()).find(|i| self.get_id(*i) == needle)
+    pub fn clear(&mut self) {
+        match self {
+            Self::Locations(array) => array.clear(),
+            Self::Uint(array) => array.clear(),
+        }
+    }
+
+    pub fn search_id(&self, needle: u32) -> Option<usize> {
+        if let Self::Uint(array) = self {
+            for (i, &u) in array.iter().enumerate() {
+                if u == needle {
+                    return Some(i);
+                }
+            }
+            None
+        } else {
+            panic!("Attempt to search for id in array of other type");
+        }
+    }
+
+    pub fn sort_ids(&mut self) {
+        if let dc_array_t::Uint(v) = self {
+            v.sort();
+        } else {
+            panic!("Attempt to sort array of something other than uints");
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const u32 {
+        if let dc_array_t::Uint(v) = self {
+            v.as_ptr()
+        } else {
+            panic!("Attempt to convert array of something other than uints to raw");
+        }
     }
 }
 
@@ -79,18 +106,6 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
-impl From<Vec<MsgId>> for dc_array_t {
-    fn from(array: Vec<MsgId>) -> Self {
-        dc_array_t::MsgIds(array)
-    }
-}
-
-impl From<Vec<ChatItem>> for dc_array_t {
-    fn from(array: Vec<ChatItem>) -> Self {
-        dc_array_t::Chat(array)
-    }
-}
-
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -103,11 +118,12 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let arr: dc_array_t = Vec::<u32>::new().into();
-        assert!(arr.len() == 0);
+        let mut arr = dc_array_t::new(7);
+        assert!(arr.is_empty());
 
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
 
         assert_eq!(arr.len(), 1000);
 
@@ -115,15 +131,31 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        assert_eq!(arr.search_id(10), Some(8));
-        assert_eq!(arr.search_id(1), None);
+        arr.clear();
+
+        assert!(arr.is_empty());
+
+        arr.add_id(13);
+        arr.add_id(7);
+        arr.add_id(666);
+        arr.add_id(0);
+        arr.add_id(5000);
+
+        arr.sort_ids();
+
+        assert_eq!(arr.get_id(0), 0);
+        assert_eq!(arr.get_id(1), 7);
+        assert_eq!(arr.get_id(2), 13);
+        assert_eq!(arr.get_id(3), 666);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let ids: Vec<u32> = (2..1002).collect();
-        let arr: dc_array_t = ids.into();
+        let mut arr = dc_array_t::new(7);
+        for i in 0..1000 {
+            arr.add_id(i + 2);
+        }
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/string.rs

@@ -105,9 +105,8 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         })
     }
 
@@ -123,9 +122,8 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
-            let std::ffi::NulError { .. } = err;
-            CStringError::InteriorNullByte
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
         }),
         None => Err(CStringError::NotUnicode),
     }
@@ -241,7 +239,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] slice can not outlive
+/// Because this returns a reference the [Path] silce can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.72"
+syn = "1.0.13"
 quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -35,11 +35,7 @@ pub fn from_sql_derive(input: TokenStream) -> TokenStream {
         impl rusqlite::types::FromSql for #name {
             fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
                 let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
+                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
             }
         }
     };

draft/group-sync.rst

@@ -5,95 +5,69 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-1. Alice and Bob are in a two-member group
+- Alice and Bob are in a two-member group
 
-2. Then Alice adds Carol, while concurrently Bob adds Doris
+- Alice adds Carol, concurrently Bob adds Doris
 
-Right now, the group has inconsistent memberships:
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
 
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
 
 
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
 
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
 
 
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
 
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
 
-For brevity let's abbreviate "group membership modification" as **GMM**.
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
 
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
+Notes:
 
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
+- mechanism works for both encrypted and unencrypted add/del messages
 
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
 
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
+- multiple member-added/removed messages can be attached in a single
+  correction message
 
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
 
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
 
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
 
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
   b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -109,3 +83,44 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

draft/msgwork_new_imap_jobs.rst

@@ -1,66 +0,0 @@
-
-simplify/streamline mark-seen/delete/move/send-mdn job handling
----------------------------------------------------------------
-
-Idea: Introduce a new "msgwork" sql table that looks very
-much like the jobs table but has a primary key "msgid"
-and no job id and no foreign-id anymore. This opens up
-bulk-processing by looking at the whole table and combining
-flag-setting to reduce imap-roundtrips and select-folder calls.
-
-Concretely, these IMAP jobs:
-
-    DeleteMsgOnImap
-    MarkseenMsgOnImap
-    MoveMsg
-
-Would be replaced by a few per-message columns in the new msgwork table:
-
-- needs_mark_seen: (bool) message shall be marked as seen on imap
-- needs_to_move: (bool) message should be moved to mvbox_folder
-- deletion_time: (target_time or 0) message shall be deleted at specified time
-- needs_send_mdn: (bool) MDN shall be sent
-
-The various places that currently add the (replaced) jobs
-would now add/modify the respective message record in the message-work table.
-
-Looking at a single message-work entry conceptually looks like this::
-
-    if msg.server_uid==0:
-        return RetryLater  # nothing can be done without server_uid
-
-    if msg.deletion_time > current_time:
-        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
-        clear(needs_deletion)
-        clear(mark_seen)
-
-    if needs_mark_seen:
-        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
-        clear(needs_mark_seen)
-
-    if needs_send_mdn:
-        schedule_smtp_send_mdn(msg)
-        clear(needs_send_mdn)
-
-    if any_flag_set():
-        retrylater
-    # remove msgwork entry from table
-
-
-Notes/Questions:
-
-- it's unclear how much we need per-message retry-time tracking/backoff
-
-- drafting bulk processing algo is useful before
-  going for the implementation, i.e. including select_folder calls etc.
-
-- maybe it's better to not have bools for the flags but
-
-  0 (no change)
-  1 (set the imap flag)
-  2 (clear the imap flag)
-
-  and design such that we can cover all imap flags.
-
-- It might not be neccessary to keep needs_send_mdn state in this table
-  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -1,30 +1,24 @@
-extern crate dirs;
-
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Error};
+use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{
-    self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
-};
+use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
+use deltachat::error::Error;
 use deltachat::imex::*;
 use deltachat::location;
-use deltachat::log::LogExt;
 use deltachat::lot::LotState;
-use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::EventType;
+use deltachat::Event;
 use deltachat::{config, provider};
-use std::fs;
-use std::time::{Duration, SystemTime};
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -92,7 +86,7 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(EventType::MsgsChanged {
+    context.emit_event(Event::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -120,11 +114,11 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         real_spec = spec.to_string();
         context
             .sql()
-            .set_raw_config("import_spec", Some(&real_spec))
+            .set_raw_config(context, "import_spec", Some(&real_spec))
             .await
             .unwrap();
     } else {
-        let rs = context.sql().get_raw_config("import_spec").await.unwrap();
+        let rs = context.sql().get_raw_config(context, "import_spec").await;
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
             return false;
@@ -163,7 +157,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(EventType::MsgsChanged {
+        context.emit_event(Event::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -175,11 +169,8 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let contact = Contact::get_by_id(context, msg.get_from_id())
         .await
         .expect("invalid contact");
-    let contact_name = if let Some(name) = msg.get_override_sender_name() {
-        format!("~{}", name)
-    } else {
-        contact.get_display_name().to_string()
-    };
+
+    let contact_name = contact.get_name();
     let contact_id = contact.get_id();
 
     let statestr = match msg.get_state() {
@@ -192,7 +183,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -200,8 +191,8 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == 1 {
+        if msg.is_starred() { "★" } else { "" },
+        if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -211,15 +202,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
-        if msg.get_viewtype() == Viewtype::VideochatInvitation {
-            format!(
-                "[VIDEOCHAT-INVITATION: {}, type={}]",
-                msg.get_videochat_url().unwrap_or_default(),
-                msg.get_videochat_type().unwrap_or_default()
-            )
-        } else {
-            "".to_string()
-        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -233,7 +215,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
+        if msg_id.is_daymarker() {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -259,11 +241,15 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
 }
 
 async fn log_contactlist(context: &Context, contacts: &[u32]) {
+    let mut contacts = contacts.to_vec();
+    if !contacts.contains(&1) {
+        contacts.push(1);
+    }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
-            let name = contact.get_display_name();
+        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
+            let name = contact.get_name();
             let addr = contact.get_addr();
             let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
@@ -289,17 +275,15 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &addr)
-                .await
-                .expect("peerstate error");
-            if peerstate.is_some() && *contact_id != 1 {
+            let peerstate = Peerstate::from_addr(context, &addr).await;
+            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
                 );
             }
 
-            println!("Contact#{}: {}{}", *contact_id, line, line2);
+            println!("Contact#{}: {}{}", contact_id, line, line2);
         }
     }
 }
@@ -359,8 +343,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listarchived\n\
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
+                 createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createprotected <name>\n\
+                 createverified <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -371,10 +356,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
+                 send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
-                 sendhtml <file for html-part> [<text for plain-part>]\n\
-                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -382,22 +366,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
-                 mute <chat-id> [<seconds>]\n\
-                 unmute <chat-id>\n\
-                 protect <chat-id>\n\
-                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
-                 ===========================Contact requests==\n\
-                 decidestartchat <msg-id>\n\
-                 decideblock <msg-id>\n\
-                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
+                 star <msg-id>\n\
+                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -406,18 +383,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  contactinfo <contact-id>\n\
                  delcontact <contact-id>\n\
                  cleanupcontacts\n\
-                 block <contact-id>\n\
-                 unblock <contact-id>\n\
-                 listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
-                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
+                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -456,21 +430,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
-            println!("Exported to {}.", dir.to_string_lossy());
+            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
+            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
-            println!("Exported to {}.", dir.to_string_lossy());
+            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -514,11 +484,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(&context).await.ok_or_log(&context);
+            sql::housekeeping(&context).await;
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
-            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
                 listflags,
@@ -526,7 +495,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -537,18 +505,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}{}",
+                        "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(&context).await?,
-                        if chat.is_muted() { "🔇" } else { "" },
+                        chat.get_id().get_fresh_msg_cnt(&context).await,
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
-                        if chat.is_protected() { "🛡️" } else { "" },
                     );
                     let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
@@ -583,11 +549,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await {
+            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
-            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -603,55 +568,34 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let time_start = std::time::SystemTime::now();
-            let msglist =
-                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
-
-            let msglist: Vec<MsgId> = msglist
-                .into_iter()
-                .map(|x| match x {
-                    ChatItem::Message { msg_id } => msg_id,
-                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
-                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
-                })
-                .collect();
-
-            let members = chat::get_chat_contacts(&context, sel_chat.id).await?;
+            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
                 let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
-            } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
-                "mailinglist".to_string()
             } else {
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}{} {}",
+                "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
                 subtitle,
-                if sel_chat.is_muted() { "🔇" } else { "" },
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
-                match sel_chat.get_profile_image(&context).await? {
+                match sel_chat.get_profile_image(&context).await {
                     Some(icon) => match icon.to_str() {
                         Some(icon) => format!(" Icon: {}", icon),
                         _ => " Icon: Err".to_string(),
                     },
                     _ => "".to_string(),
                 },
-                if sel_chat.is_protected() {
-                    "🛡️"
-                } else {
-                    ""
-                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -660,77 +604,47 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!(
                 "{} messages.",
-                sel_chat.get_id().get_msg_cnt(&context).await?
+                sel_chat.get_id().get_msg_cnt(&context).await
             );
-
-            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
-            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
-
-            println!(
-                "{:?} to create this list, {:?} to mark all messages as noticed.",
-                time_needed, time_noticed_needed
-            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: u32 = arg1.parse()?;
-            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
+            let contact_id: libc::c_int = arg1.parse()?;
+            let chat_id = chat::create_by_contact_id(&context, contact_id as u32).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
-        "decidestartchat" | "createchatbymsg" => {
+        "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
             let msg_id = MsgId::new(arg1.parse()?);
-            match message::decide_on_contact_request(
-                &context,
-                msg_id,
-                ContactRequestDecision::StartChat,
-            )
-            .await
-            {
-                Some(chat_id) => {
-                    let chat = Chat::load_from_db(&context, chat_id).await?;
-                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
-                }
-                None => println!("Cannot crate chat."),
-            }
-        }
-        "decidenotnow" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id = MsgId::new(arg1.parse()?);
-            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
-                .await;
-        }
-        "decideblock" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id = MsgId::new(arg1.parse()?);
-            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
-                .await;
+            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
+            let chat = Chat::load_from_db(&context, chat_id).await?;
+
+            println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
+                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createprotected" => {
+        "createverified" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id =
-                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
+            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
 
-            println!("Group#{} created and protected successfully.", chat_id);
+            println!("VerifiedGroup#{} created successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: u32 = arg1.parse()?;
+            let contact_id_0: libc::c_int = arg1.parse()?;
             if chat::add_contact_to_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_0,
+                contact_id_0 as u32,
             )
             .await
             {
@@ -742,11 +656,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: u32 = arg1.parse()?;
+            let contact_id_1: libc::c_int = arg1.parse()?;
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_1,
+                contact_id_1 as u32,
             )
             .await?;
 
@@ -772,7 +686,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts =
-                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await;
             println!("Memberlist:");
 
             log_contactlist(&context, &contacts).await;
@@ -781,7 +695,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    Some(sel_chat.as_ref().unwrap().get_id())
+                    sel_chat.as_ref().unwrap().get_id()
                 )
                 .await,
             );
@@ -789,15 +703,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id: Option<u32> = arg1.parse().ok();
+            let contact_id = arg1.parse().unwrap_or_default();
             let locations = location::get_range(
                 &context,
-                Some(sel_chat.as_ref().unwrap().get_id()),
+                sel_chat.as_ref().unwrap().get_id(),
                 contact_id,
                 0,
                 0,
             )
-            .await?;
+            .await;
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
@@ -880,42 +794,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
-        "sendhtml" => {
-            ensure!(sel_chat.is_some(), "No chat selected.");
-            ensure!(!arg1.is_empty(), "No html-file given.");
-            let path: &Path = arg1.as_ref();
-            let html = &*fs::read(&path)?;
-            let html = String::from_utf8_lossy(html);
-
-            let mut msg = Message::new(Viewtype::Text);
-            msg.set_html(Some(html.to_string()));
-            msg.set_text(Some(if arg2.is_empty() {
-                path.file_name().unwrap().to_string_lossy().to_string()
-            } else {
-                arg2.to_string()
-            }));
-            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
-        }
-        "videochat" => {
-            ensure!(sel_chat.is_some(), "No chat selected.");
-            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
-        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                Some(sel_chat.get_id())
+                sel_chat.get_id()
             } else {
-                None
+                ChatId::new(0)
             };
 
-            let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
+            let msglist = context.search_msgs(chat, arg1).await;
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
-            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -928,7 +819,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, Some(&mut draft))
-                    .await?;
+                    .await;
                 println!("Draft saved.");
             } else {
                 sel_chat
@@ -936,7 +827,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, None)
-                    .await?;
+                    .await;
                 println!("Draft deleted.");
             }
         }
@@ -949,6 +840,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
+        "updatedevicechats" => {
+            context.update_device_chats().await?;
+        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -959,7 +853,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Viewtype::Gif,
                 Viewtype::Video,
             )
-            .await?;
+            .await;
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
@@ -980,39 +874,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => unreachable!("arg0={:?}", arg0),
-                    },
-                )
-                .await?;
-        }
-        "mute" | "unmute" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            let duration = match arg0 {
-                "mute" => {
-                    if arg2.is_empty() {
-                        MuteDuration::Forever
-                    } else {
-                        SystemTime::now()
-                            .checked_add(Duration::from_secs(arg2.parse()?))
-                            .map_or(MuteDuration::Forever, MuteDuration::Until)
-                    }
-                }
-                "unmute" => MuteDuration::NotMuted,
-                _ => unreachable!("arg0={:?}", arg0),
-            };
-            chat::set_muted(&context, chat_id, duration).await?;
-        }
-        "protect" | "unprotect" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id
-                .set_protection(
-                    &context,
-                    match arg0 {
-                        "protect" => ProtectionStatus::Protected,
-                        "unprotect" => ProtectionStatus::Unprotected,
-                        _ => unreachable!("arg0={:?}", arg0),
+                        _ => panic!("Unexpected command (This should never happen)"),
                     },
                 )
                 .await?;
@@ -1025,21 +887,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            let res = message::get_msg_info(&context, id).await?;
+            let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
-        "html" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            let file = dirs::home_dir()
-                .unwrap_or_default()
-                .join(format!("msg-{}.html", id.to_u32()));
-            let html = id.get_html(&context).await?.unwrap_or_default();
-            fs::write(&file, html)?;
-            println!("HTML written to: {:#?}", file);
-        }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs().await?;
+            let msglist = context.get_fresh_msgs().await;
 
             log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
@@ -1059,7 +911,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await?;
+            message::markseen_msgs(&context, msg_ids).await;
+        }
+        "star" | "unstar" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let mut msg_ids = vec![MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            message::star_msgs(&context, msg_ids, arg0 == "star").await;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -1071,9 +929,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contacts = Contact::get_all(
                 &context,
                 if arg0 == "listverified" {
-                    DC_GCL_VERIFIED_ONLY | DC_GCL_ADD_SELF
+                    0x1 | 0x2
                 } else {
-                    DC_GCL_ADD_SELF
+                    0x2
                 },
                 Some(arg1),
             )
@@ -1086,7 +944,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, &book).await?;
+                Contact::add_address_book(&context, book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }
@@ -1094,14 +952,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = arg1.parse()?;
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
             let mut res = format!(
                 "Contact info for: {}:\nIcon: {}\n",
                 name_n_addr,
-                match contact.get_profile_image(&context).await? {
+                match contact.get_profile_image(&context).await {
                     Some(image) => image.to_str().unwrap().to_string(),
                     None => "NoIcon".to_string(),
                 }
@@ -1131,21 +989,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             Contact::delete(&context, arg1.parse()?).await?;
         }
-        "block" => {
-            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
-            Contact::block(&context, contact_id).await;
-        }
-        "unblock" => {
-            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
-            Contact::unblock(&context, contact_id).await;
-        }
-        "listblocked" => {
-            let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await;
-            println!("{} blocked contacts.", contacts.len());
-        }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let res = check_qr(&context, arg1).await;
@@ -1166,7 +1009,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            match provider::get_provider_info(arg1).await {
+            match provider::get_provider_info(arg1) {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);
@@ -1186,11 +1029,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
-        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
+        //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r,
+        //         event, event as usize, r as libc::c_int,
         //     );
         // }
         "fileinfo" => {
@@ -1213,6 +1056,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 seconds, device_cnt, server_cnt
             );
         }
+        "emptyserver" => {
+            ensure!(!arg1.is_empty(), "Argument <flags> missing");
+
+            message::dc_empty_server(&context, arg1.parse()?).await;
+        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -19,52 +19,50 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
-use log::{error, info, warn};
+use deltachat::Event;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
 
 /// Event Handler
-fn receive_event(event: EventType) {
+fn receive_event(event: Event) {
     let yellow = Color::Yellow.normal();
+    let red = Color::Red.normal();
     match event {
-        EventType::Info(msg) => {
-            /* do not show the event as this would fill the screen */
-            info!("{}", msg);
+        Event::Info(msg) => {
+            println!("[INFO] {}", msg);
         }
-        EventType::SmtpConnected(msg) => {
-            info!("[SMTP_CONNECTED] {}", msg);
+        Event::SmtpConnected(msg) => {
+            println!("[INFO SMTP_CONNECTED] {}", msg);
         }
-        EventType::ImapConnected(msg) => {
-            info!("[IMAP_CONNECTED] {}", msg);
+        Event::ImapConnected(msg) => {
+            println!("[INFO IMAP_CONNECTED] {}", msg);
         }
-        EventType::SmtpMessageSent(msg) => {
-            info!("[SMTP_MESSAGE_SENT] {}", msg);
+        Event::SmtpMessageSent(msg) => {
+            println!("[INFO SMTP_MESSAGE_SENT] {}", msg);
         }
-        EventType::Warning(msg) => {
-            warn!("{}", msg);
+        Event::Warning(msg) => {
+            println!("[WARNING] {}", msg);
         }
-        EventType::Error(msg) => {
-            error!("{}", msg);
+        Event::Error(msg) => {
+            println!("[ERROR] {}", red.paint(msg));
         }
-        EventType::ErrorNetwork(msg) => {
-            error!("[NETWORK] msg={}", msg);
+        Event::ErrorNetwork(msg) => {
+            println!("[ERROR NETWORK] msg={}", red.paint(msg));
         }
-        EventType::ErrorSelfNotInGroup(msg) => {
-            error!("[SELF_NOT_IN_GROUP] {}", msg);
+        Event::ErrorSelfNotInGroup(msg) => {
+            println!("[ERROR SELF_NOT_IN_GROUP] {}", red.paint(msg));
         }
-        EventType::MsgsChanged { chat_id, msg_id } => {
-            info!(
+        Event::MsgsChanged { chat_id, msg_id } => {
+            println!(
                 "{}",
                 yellow.paint(format!(
                     "Received MSGS_CHANGED(chat_id={}, msg_id={})",
@@ -72,51 +70,41 @@ fn receive_event(event: EventType) {
                 ))
             );
         }
-        EventType::ContactsChanged(_) => {
-            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
+        Event::ContactsChanged(_) => {
+            println!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        EventType::LocationChanged(contact) => {
-            info!(
+        Event::LocationChanged(contact) => {
+            println!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        EventType::ConfigureProgress { progress, comment } => {
-            if let Some(comment) = comment {
-                info!(
-                    "{}",
-                    yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
-                    ))
-                );
-            } else {
-                info!(
-                    "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-                );
-            }
+        Event::ConfigureProgress(progress) => {
+            println!(
+                "{}",
+                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+            );
         }
-        EventType::ImexProgress(progress) => {
-            info!(
+        Event::ImexProgress(progress) => {
+            println!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        EventType::ImexFileWritten(file) => {
-            info!(
+        Event::ImexFileWritten(file) => {
+            println!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        EventType::ChatModified(chat) => {
-            info!(
+        Event::ChatModified(chat) => {
+            println!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
             );
         }
         _ => {
-            info!("Received {:?}", event);
+            println!("Received {}", yellow.paint(format!("{:?}", event)));
         }
     }
 }
@@ -169,14 +157,12 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 34] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
-    "decidestartchat",
-    "decideblock",
-    "decidenotnow",
+    "createchatbymsg",
     "creategroup",
     "createverified",
     "addmember",
@@ -191,38 +177,31 @@ const CHAT_COMMANDS: [&str; 34] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
-    "videochat",
     "draft",
     "listmedia",
     "archive",
     "unarchive",
     "pin",
     "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 6] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
 const MISC_COMMANDS: [&str; 10] = [
     "getqr",
@@ -238,9 +217,7 @@ const MISC_COMMANDS: [&str; 10] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -262,10 +239,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -286,19 +264,18 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event.typ);
+            receive_event(event);
         }
     });
 
@@ -311,7 +288,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
+    let (reader_s, reader_r) = async_std::sync::channel(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -320,8 +297,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
@@ -334,7 +311,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    async_std::task::block_on(reader_s.send(line));
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -393,7 +370,7 @@ async fn handle_cmd(
             ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
                 let oauth2_url =
                     dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
@@ -411,8 +388,9 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -430,7 +408,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                dc_join_securejoin(&ctx, arg1).await;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -1,25 +1,25 @@
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::Event;
 
-fn cb(event: EventType) {
+fn cb(event: Event) {
     match event {
-        EventType::ConfigureProgress { progress, .. } => {
+        Event::ConfigureProgress(progress) => {
             log::info!("progress: {}", progress);
         }
-        EventType::Info(msg) => {
+        Event::Info(msg) => {
             log::info!("{}", msg);
         }
-        EventType::Warning(msg) => {
+        Event::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
+        Event::Error(msg) | Event::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
+    let ctx = Context::new("FakeOs".into(), dbfile.into())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -45,7 +45,7 @@ async fn main() {
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event.typ);
+            cb(event);
         }
     });
 
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);

python/CHANGELOG

@@ -1,29 +1,6 @@
-1.51.0
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
-1.44.0
-------
-
-- fix Chat.get_mute_duration()
-
-1.40.1
+0.900.0 (DRAFT)
 ---------------
 
-- emit "ac_member_removed" event (with 'actor' being the removed contact) 
-  for when a user leaves a group.
-
-- fix create_contact(addr) when addr is the self-contact. 
-
-
-1.40.0
----------------
-
-- uses latest 1.40+ Delta Chat core
-
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -33,7 +10,6 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
-
 0.800.0
 -------
 

python/README.rst

@@ -7,14 +7,76 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -41,78 +103,6 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Running tests
-=============
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-This will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _livetests:
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
-
-    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
-
-With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
-These accounts exists for one 1hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
-
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
-
-Each test run creates new accounts.
-
-
-.. _sourceinstall:
-
-Installing bindings from source (Updated: July 2020)
-=========================================================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   python install_python_bindings.py
-
-The core compilation and bindings building might take a while,
-depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
 Code examples
 =============
 
@@ -142,7 +132,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -151,9 +141,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/doc/_templates/globaltoc.html

@@ -9,7 +9,8 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
+  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, actor, message):
+    def ac_member_added(self, chat, contact, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, actor, message):
+    def ac_member_removed(self, chat, contact, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+            contact.addr, chat.id, message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    bot_chat = bot_contact.create_chat()
-    bot_chat.send_text("hello")
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
 
-    lp.sec("waiting for the reply message from the bot to arrive")
+    lp.sec("waiting for the bot-reply to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.chat == bot_chat
     assert "hello" in reply.text
+    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    bot_chat.send_text("/quit")
+    ch1.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1))
-    ac2.add_account_plugin(FFIEventLogger(ac2))
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_added {}*
+    """.format(contact3.addr))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -17,11 +17,8 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-
     if target == 'release':
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
-
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,8 +60,7 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
+        log = events.FFIEventLogger(ac, "bot")
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -77,9 +76,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
+        ac.configure()
+        ac.wait_configure_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,63 +1,65 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import os
-import platform
-import re
-import shutil
-import subprocess
 import tempfile
-import textwrap
-import types
-
+import platform
+import os
 import cffi
+import shutil
+from os.path import dirname as dn
+from os.path import abspath
 
 
-def local_build_flags(projdir, target):
-    """Construct build flags for building against a checkout.
-
-    :param projdir: The root directory of the deltachat-core-rust project.
-    :param target: The rust build target, `debug` or `release`.
-    """
-    flags = types.SimpleNamespace()
-    if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
-        ]
-    elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        if platform.system() == 'Darwin':
+            libs = ['resolv', 'dl']
+            extra_link_args = [
+                    '-framework', 'CoreFoundation',
+                    '-framework', 'CoreServices',
+                    '-framework', 'Security',
+            ]
+        elif platform.system() == 'Linux':
+            libs = ['rt', 'dl', 'm']
+            extra_link_args = []
+        else:
+            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
-        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-    target_dir = os.environ.get("CARGO_TARGET_DIR")
-    if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
-    return flags
-
-
-def system_build_flags():
-    """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-    return flags
-
-
-def extract_functions(flags):
-    """Extract the function definitions from deltachat.h.
-
-    This creates a .h file with a single `#include <deltachat.h>` line
-    in it.  It then runs the C preprocessor to create an output file
-    which contains all function definitions found in `deltachat.h`.
-    """
+        libs = ['deltachat']
+        objs = []
+        incs = []
+        extra_link_args = []
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
+        extra_link_args=extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -69,133 +71,13 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            return dst_fp.read()
+            builder.cdef(dst_fp.read())
     finally:
         shutil.rmtree(tmpdir)
 
-
-def find_header(flags):
-    """Use the compiler to find the deltachat.h header location.
-
-    This uses a small utility in deltachat.h to find the location of
-    the header file location.
-    """
-    distutils.log.set_verbosity(distutils.log.INFO)
-    cc = distutils.ccompiler.new_compiler(force=True)
-    distutils.sysconfig.customize_compiler(cc)
-    tmpdir = tempfile.mkdtemp()
-    try:
-        src_name = os.path.join(tmpdir, "where.c")
-        obj_name = os.path.join(tmpdir, "where.o")
-        dst_name = os.path.join(tmpdir, "where")
-        with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
-                #include <stdio.h>
-                #include <deltachat.h>
-
-                int main(void) {
-                    printf("%s", _dc_header_file_location());
-                    return 0;
-                }
-            """))
-        cwd = os.getcwd()
-        try:
-            os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
-                       macros=[("PY_CFFI_INC", "1")])
-        finally:
-            os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
-        return subprocess.check_output(dst_name)
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-def extract_defines(flags):
-    """Extract the required #DEFINEs from deltachat.h.
-
-    Since #DEFINEs are interpreted by the C preprocessor we can not
-    use the compiler to extract these and need to parse the header
-    file ourselves.
-
-    The defines are returned in a string that can be passed to CFFIs
-    cdef() method.
-    """
-    header = find_header(flags)
-    defines_re = re.compile(r"""
-        \#define\s+   # The start of a define.
-        (             # Begin capturing group which captures the define name.
-            (?:       # A nested group which is not captured, this allows us
-                      #    to build the list of prefixes to extract without
-                      #    creation another capture group.
-                DC_EVENT
-                | DC_QR
-                | DC_MSG
-                | DC_LP
-                | DC_EMPTY
-                | DC_CERTCK
-                | DC_STATE
-                | DC_STR
-                | DC_CONTACT_ID
-                | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
-                | DC_CHAT
-                | DC_PROVIDER
-                | DC_KEY_GEN
-                | DC_IMEX
-            )         # End of prefix matching
-            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
-        )             # Close the capturing group, this contains
-                      # the entire name e.g. DC_MSG_TEXT.
-        \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
-    defines = []
-    with open(header) as fp:
-        for line in fp:
-            match = defines_re.match(line)
-            if match:
-                defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
-
-
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
-        flags = local_build_flags(projdir, target)
-    else:
-        flags = system_build_flags()
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
-    function_defs = extract_functions(flags)
-    defines = extract_defines(flags)
-    builder.cdef(function_defs)
-    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -89,22 +89,6 @@ class Account(object):
             d[key.lower()] = value
         return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 
@@ -195,6 +179,17 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -218,48 +213,24 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, email, name=None):
+        """ create a (new) Contact. If there already is a Contact
+        with that e-mail address, it is unblocked and its name is
+        updated.
 
-        Calling this method will always result in the same
-        underlying contact id.  If there already is a Contact
-        with that e-mail address, it is unblocked and its display
-        `name` is updated if specified.
-
-        :param obj: email-address, Account or Contact instance.
-        :param name: (optional) display name for this contact
+        :param email: email-address (text type)
+        :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return Contact(self, contact_id)
 
-    def get_contact(self, obj):
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name=None):
-        if isinstance(obj, Account):
-            if not obj.is_configured():
-                raise ValueError("can only add addresses from configured accounts")
-            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
-        elif isinstance(obj, Contact):
-            if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
-            addr, displayname = obj.addr, obj.name
-        elif isinstance(obj, str):
-            displayname, addr = parseaddr(obj)
-        else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
-
-        if name is None and displayname:
-            name = displayname
-        return (name, addr)
-
     def delete_contact(self, contact):
         """ delete a Contact.
 
@@ -279,24 +250,6 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
-    def get_blocked_contacts(self):
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -326,29 +279,53 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
-        return self.create_contact(obj).create_chat()
+    def create_chat_by_contact(self, contact):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
-    def _create_chat_by_message_id(self, msg_id):
-        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(contact, "id"):
+            if contact.account != self:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
+        return Chat(self, chat_id)
 
-    def create_group_chat(self, name, contacts=None, verified=False):
+    def create_chat_by_message(self, message):
+        """ create or get an existing chat object for the
+        the specified message.
+
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
+        :param message: messsage id or message instance.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(message, "id"):
+            if message.account != self:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
+        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
+        return Chat(self, chat_id)
+
+    def create_group_chat(self, name, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
-        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        chat = Chat(self, chat_id)
-        if contacts is not None:
-            for contact in contacts:
-                chat.add_contact(contact)
-        return chat
+        return Chat(self, chat_id)
 
     def get_chats(self):
         """ return list of chats.
@@ -370,9 +347,6 @@ class Account(object):
     def get_deaddrop_chat(self):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
-    def get_device_chat(self):
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
-
     def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
@@ -380,6 +354,13 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -428,23 +409,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account has to be stopped; call stop_io() if necessary.
+        Note that the account does not have to be started.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -454,7 +435,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
+        self._import(path, imex_cmd=2)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -462,22 +443,21 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -584,52 +564,35 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
-        If you are using this from a test, you may want to call
-        wait_all_initial_fetches() afterwards.
-
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None
+        :returns: None (account is configured and with io-scheduling running)
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def maybe_network(self):
-        """This function should be called when there is a hint
-        that the network is available again,
-        e.g. as a response to system event reporting network availability.
-        The library will try to send pending messages out immediately.
-
-        Moreover, to have a reliable state
-        when the app comes to foreground with network available,
-        it may be reasonable to call the function also at that moment.
-
-        It is okay to call the function unconditionally when there is
-        network available, however, calling the function
-        _without_ having network may interfere with the backoff algorithm
-        and will led to let the jobs fail faster, with fewer retries
-        and may avoid messages being sent out.
-
-        Finally, if the context was created by the dc_accounts_t account manager
-        (currently not implemented in the Python bindings),
-        use dc_accounts_maybe_network() instead of this function
-        """
-        lib.dc_maybe_network(self._dc_context)
-
-    def configure(self, reconfigure=False):
-        """ Start configuration process and return a Configtracker instance
-        on which you can block with wait_finish() to get a True/False success
-        value for the configuration process.
-        """
-        assert self.is_configured() == reconfigure
+    def configure(self):
+        assert not self.is_configured()
+        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        configtracker = ConfigureTracker(self)
-        self.add_account_plugin(configtracker)
+        if hasattr(self, "_configtracker"):
+            self.remove_account_plugin(self._configtracker)
+        self._configtracker = ConfigureTracker()
+        self.add_account_plugin(self._configtracker)
         lib.dc_configure(self._dc_context)
-        return configtracker
+
+    def wait_configure_finish(self):
+        try:
+            self._configtracker.wait_finish()
+        finally:
+            self.remove_account_plugin(self._configtracker)
+            del self._configtracker
+
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
 
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
@@ -640,8 +603,11 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        self.log("dc_stop_io (stop core IO scheduler)")
-        lib.dc_stop_io(self._dc_context)
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove
@@ -652,24 +618,12 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-
-        # if _dc_context is unref'ed the event thread should quickly
-        # receive the termination signal. However, some python code might
-        # still hold a reference and so we use a secondary signal
-        # to make sure the even thread terminates if it receives any new
-        # event, indepedently from waiting for the core to send NULL to
-        # get_next_event().
-        self._event_thread.mark_shutdown()
+        # the dc_context_unref triggers get_next_event to return ffi.NULL
+        # which in turns makes the event thread finish execution
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        try:
-            self._event_thread.wait(timeout=2)
-        except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
-
-        if self._event_thread.is_alive():
-            self.log("WARN: event thread did not terminate yet, ignoring.")
+        self._event_thread.wait()
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -18,8 +18,6 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -57,7 +55,10 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
 
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
@@ -82,20 +83,12 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self):
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the deaddrop or for the device-talk
+    def is_verified(self):
+        """ return True if this chat is a verified group.
 
-        :returns: True if the chat is writable, False otherwise
+        :returns: True if chat is verified, False otherwise.
         """
-        return lib.dc_chat_can_send(self._dc_chat)
-
-    def is_protected(self):
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
+        return lib.dc_chat_is_verified(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.
@@ -142,23 +135,7 @@ class Chat(object):
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
-
-    def get_ephemeral_timer(self):
-        """ get ephemeral timer.
-
-        :returns: ephemeral timer value in seconds
-        """
-        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
-
-    def set_ephemeral_timer(self, timer):
-        """ set ephemeral timer.
-
-        :param: timer value in seconds
-
-        :returns: None
-        """
-        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
     def get_type(self):
         """ (deprecated) return type of this chat.
@@ -167,13 +144,6 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self):
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -254,19 +224,17 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """ create a new prepared message.
 
         :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
+        :returns: :class:`deltachat.message.Message` instance.
         """
         msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -360,34 +328,33 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, obj):
+    def add_contact(self, contact):
         """ add a contact to this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
-        return contact
 
-    def remove_contact(self, obj):
+    def remove_contact(self, contact):
         """ remove a contact from this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
+        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
+
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -398,14 +365,6 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
-    def num_contacts(self):
-        """ return number of contacts in this chat. """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return lib.dc_array_get_cnt(dc_array)
-
     def set_profile_image(self, img_path):
         """Set group profile image.
 
@@ -504,23 +463,18 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i)
-                ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
-        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -1,7 +1,197 @@
-from .capi import lib
+import sys
+import re
+import os
+from os.path import dirname, abspath
+from os.path import join as joinpath
+
+# the following const are generated from deltachat.h
+# this works well when you in a git-checkout
+# run "python deltachat/const.py" to regenerate events
+# begin const generated
+DC_GCL_ARCHIVED_ONLY = 0x01
+DC_GCL_NO_SPECIALS = 0x02
+DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_FOR_FORWARDING = 0x08
+DC_GCL_VERIFIED_ONLY = 0x01
+DC_GCL_ADD_SELF = 0x02
+DC_QR_ASK_VERIFYCONTACT = 200
+DC_QR_ASK_VERIFYGROUP = 202
+DC_QR_FPR_OK = 210
+DC_QR_FPR_MISMATCH = 220
+DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
+DC_QR_ADDR = 320
+DC_QR_TEXT = 330
+DC_QR_URL = 332
+DC_QR_ERROR = 400
+DC_CHAT_ID_DEADDROP = 1
+DC_CHAT_ID_TRASH = 3
+DC_CHAT_ID_MSGS_IN_CREATION = 4
+DC_CHAT_ID_STARRED = 5
+DC_CHAT_ID_ARCHIVED_LINK = 6
+DC_CHAT_ID_ALLDONE_HINT = 7
+DC_CHAT_ID_LAST_SPECIAL = 9
+DC_CHAT_TYPE_UNDEFINED = 0
+DC_CHAT_TYPE_SINGLE = 100
+DC_CHAT_TYPE_GROUP = 120
+DC_CHAT_TYPE_VERIFIED_GROUP = 130
+DC_MSG_ID_MARKER1 = 1
+DC_MSG_ID_DAYMARKER = 9
+DC_MSG_ID_LAST_SPECIAL = 9
+DC_STATE_UNDEFINED = 0
+DC_STATE_IN_FRESH = 10
+DC_STATE_IN_NOTICED = 13
+DC_STATE_IN_SEEN = 16
+DC_STATE_OUT_PREPARING = 18
+DC_STATE_OUT_DRAFT = 19
+DC_STATE_OUT_PENDING = 20
+DC_STATE_OUT_FAILED = 24
+DC_STATE_OUT_DELIVERED = 26
+DC_STATE_OUT_MDN_RCVD = 28
+DC_CONTACT_ID_SELF = 1
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
+DC_CONTACT_ID_LAST_SPECIAL = 9
+DC_MSG_TEXT = 10
+DC_MSG_IMAGE = 20
+DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
+DC_MSG_AUDIO = 40
+DC_MSG_VOICE = 41
+DC_MSG_VIDEO = 50
+DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
+DC_EVENT_INFO = 100
+DC_EVENT_SMTP_CONNECTED = 101
+DC_EVENT_IMAP_CONNECTED = 102
+DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
+DC_EVENT_WARNING = 300
+DC_EVENT_ERROR = 400
+DC_EVENT_ERROR_NETWORK = 401
+DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
+DC_EVENT_MSGS_CHANGED = 2000
+DC_EVENT_INCOMING_MSG = 2005
+DC_EVENT_MSG_DELIVERED = 2010
+DC_EVENT_MSG_FAILED = 2012
+DC_EVENT_MSG_READ = 2015
+DC_EVENT_CHAT_MODIFIED = 2020
+DC_EVENT_CONTACTS_CHANGED = 2030
+DC_EVENT_LOCATION_CHANGED = 2035
+DC_EVENT_CONFIGURE_PROGRESS = 2041
+DC_EVENT_IMEX_PROGRESS = 2051
+DC_EVENT_IMEX_FILE_WRITTEN = 2052
+DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
+DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_FILE_COPIED = 2055
+DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
+# end const generated
 
 
-for name in dir(lib):
-    if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+def read_event_defines(f):
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
+    for line in f:
+        m = rex.match(line)
+        if m:
+            yield m.groups()
+
+
+if __name__ == "__main__":
+    here = abspath(__file__).rstrip("oc")
+    here_dir = dirname(here)
+    if len(sys.argv) >= 2:
+        deltah = sys.argv[1]
+    else:
+        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
+    assert os.path.exists(deltah)
+
+    lines = []
+    skip_to_end = False
+    for orig_line in open(here):
+        if skip_to_end:
+            if not orig_line.startswith("# end const"):
+                continue
+            skip_to_end = False
+        lines.append(orig_line)
+        if orig_line.startswith("# begin const"):
+            with open(deltah) as f:
+                for name, item in read_event_defines(f):
+                    lines.append("{} = {}\n".format(name, item))
+            skip_to_end = True
+
+    tmpname = here + ".tmp"
+    with open(tmpname, "w") as f:
+        f.write("".join(lines))
+    os.rename(tmpname, here)

python/src/deltachat/contact.py

@@ -3,8 +3,6 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
-from .chat import Chat
-from . import const
 
 
 class Contact(object):
@@ -13,8 +11,6 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -40,29 +36,14 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self):
+    def display_name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
-    # deprecated alias
-    display_name = name
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
-    def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, block)
-
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
@@ -77,24 +58,6 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
-
-    def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        dc_context = self.account._dc_context
-        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
-        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
-        return Chat(self.account, chat_id)
-
-    # deprecated name
-    get_chat = create_chat
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/cutil.py

@@ -17,8 +17,7 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/src/deltachat/direct_imap.py

@@ -1,257 +0,0 @@
-"""
-Internal Python-level IMAP handling used by the testplugin
-and for cleaning up inbox/mvbox for each test function run.
-"""
-
-import io
-import email
-import ssl
-import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
-import imaplib
-import deltachat
-from deltachat import const
-
-
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
-ALL = "1:*"
-
-
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.delete_folder(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
-
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
-class DirectImap:
-    def __init__(self, account):
-        self.account = account
-        self.logid = account.get_config("displayname") or id(account)
-        self._idling = False
-        self.connect()
-
-    def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
-
-        user = self.account.get_config("addr")
-        pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
-        self.conn.login(user, pw)
-
-        self.select_folder("INBOX")
-
-    def shutdown(self):
-        try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
-            pass
-        try:
-            self.conn.logout()
-        except (OSError, IMAPClientError):
-            print("Could not logout direct_imap conn")
-
-    def create_folder(self, foldername):
-        try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
-    def select_folder(self, foldername):
-        assert not self._idling
-        return self.conn.select_folder(foldername)
-
-    def select_config_folder(self, config_name):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
-        if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
-        foldername = self.account.get_config(config_name)
-        if foldername:
-            return self.select_folder(foldername)
-
-    def list_folders(self):
-        """ return list of all existing folder names"""
-        assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
-
-    def delete(self, range, expunge=True):
-        """ delete a range of messages (imap-syntax).
-        If expunge is true, perform the expunge-operation
-        to make sure the messages are really gone and not
-        just flagged as deleted.
-        """
-        self.conn.set_flags(range, [DELETED])
-        if expunge:
-            self.conn.expunge()
-
-    def get_all_messages(self):
-        assert not self._idling
-
-        # Flush unsolicited responses. IMAPClient has problems
-        # dealing with them: https://github.com/mjs/imapclient/issues/334
-        # When this NOOP was introduced, next FETCH returned empty
-        # result instead of a single message, even though IMAP server
-        # can only return more untagged responses than required, not
-        # less.
-        self.conn.noop()
-
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
-        assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
-
-    def mark_all_read(self):
-        messages = self.get_unread_messages()
-        if messages:
-            res = self.conn.set_flags(messages, [SEEN])
-            print("marked seen:", messages, res)
-
-    def get_unread_cnt(self):
-        return len(self.get_unread_messages())
-
-    def dump_imap_structures(self, dir, logfile):
-        assert not self._idling
-        stream = io.StringIO()
-
-        def log(*args, **kwargs):
-            kwargs["file"] = stream
-            print(*args, **kwargs)
-
-        empty_folders = []
-        for imapfolder in self.list_folders():
-            self.select_folder(imapfolder)
-            messages = list(self.get_all_messages())
-            if not messages:
-                empty_folders.append(imapfolder)
-                continue
-
-            log("---------", imapfolder, len(messages), "messages ---------")
-            # get message content without auto-marking it as seen
-            # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[]']
-                if not body_bytes:
-                    log("Message", uid, "has empty body")
-                    continue
-
-                flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
-                path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
-
-        if empty_folders:
-            log("--------- EMPTY FOLDERS:", empty_folders)
-
-        print(stream.getvalue(), file=logfile)
-
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False):
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
-        if terminate:
-            self.idle_done()
-        self.account.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
-        """
-        while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle_done()
-            self._idling = False
-            return res
-
-    def append(self, folder, msg):
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)

python/src/deltachat/events.py

@@ -1,7 +1,6 @@
 import threading
 import time
 import re
-import os
 from queue import Queue, Empty
 
 import deltachat
@@ -29,9 +28,13 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account):
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
         self.account = account
-        self.logid = self.account.get_config("displayname")
+        self.logid = logid
         self.init_time = time.time()
 
     @account_hookimpl
@@ -49,15 +52,6 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-
-        if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
-            if message.startswith("DC_EVENT_WARNING"):
-                s = WARN + s + ENDC
-            if message.startswith("DC_EVENT_ERROR"):
-                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -96,21 +90,13 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex):
-        rex = re.compile(regex)
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.search(ev.data2):
+            if rex.match(ev.data2):
                 return ev
 
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -129,15 +115,6 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_all_initial_fetches(self):
-        """Has to be called after start_io() to wait for fetch_existing_msgs to run
-        so that new messages are not mistaken for old ones:
-        - ac1 and ac2 are created
-        - ac1 sends a message to ac2
-        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("Done fetching existing messages")
-
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
@@ -150,12 +127,6 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
-    def wait_msg_delivered(self, msg):
-        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == msg.chat.id
-        assert ev.data2 == msg.id
-        assert msg.is_out_delivered()
-
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.
@@ -166,7 +137,6 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
-        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -175,15 +145,12 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self):
-        self._marked_for_shutdown = True
-
-    def wait(self, timeout=None):
+    def wait(self):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join(timeout=timeout)
+        self.join()
 
     def run(self):
         """ get and run events until shutdown. """
@@ -195,12 +162,10 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while not self._marked_for_shutdown:
+        while 1:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
-            if self._marked_for_shutdown:
-                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are generally not allowed to block/last long as this
+    Hooks are not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,6 +31,10 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -39,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
+        """ Called when a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -51,37 +55,19 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
-
-        :param message: Message that was just delivered.
-        """
+        """ Called when an outgoing message has been delivered to SMTP. """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
-
-        :param chat: Chat which was modified.
-        """
+        """ Chat was created or modified regarding membership, avatar, title. """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
-
-        :param chat: Chat where contact was added.
-        :param contact: Contact that was added.
-        :param actor: Who added the contact (None if it was our self-addr)
-        :param message: The original system message that reports the addition.
-        """
+    def ac_member_added(self, chat, contact, message):
+        """ Called for each contact added to an accepted chat. """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
-
-        :param chat: Chat where contact was removed.
-        :param contact: Contact that was removed.
-        :param actor: Who removed the contact (None if it was our self-addr)
-        :param message: The original system message that reports the removal.
-        """
+    def ac_member_removed(self, chat, contact, message):
+        """ Called for each contact removed from a chat.  """
 
 
 class Global:
@@ -102,14 +88,6 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
-    @global_hookspec
-    def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
-
-        This hook can be used to perform extra work before
-        ac_configure_completed is called.
-        """
-
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -1,7 +1,6 @@
 """ The Message object. """
 
 import os
-import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -21,8 +20,8 @@ class Message(object):
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        self.id = lib.dc_msg_get_id(dc_msg)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
@@ -46,36 +45,23 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param: view_type is "text", "audio", "video", "file"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
         ))
 
-    def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
-
-        If the message is a deaddrop contact request
-        the sender will become an accepted contact.
-
-        :returns: a :class:`deltachat.chat.Chat` object.
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
         """
-        from .chat import Chat
-        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
-        ctx = self.account._dc_context
-        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
-        return Chat(self.account, chat_id)
-
-    @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self.account._dc_context, self.id),
+                lib.dc_msg_unref
+        )
 
     @props.with_doc
     def text(self):
@@ -86,23 +72,6 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self):
-        """html text of this messages (might be empty if not an html message). """
-        return from_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -181,47 +150,6 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
-    @props.with_doc
-    def ephemeral_timer(self):
-        """Ephemeral timer in seconds
-
-        :returns: timer in seconds or None if there is no timer
-        """
-        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
-        if timer:
-            return timer
-
-    @props.with_doc
-    def ephemeral_timestamp(self):
-        """UTC time when the message will be deleted.
-
-        :returns: naive datetime.datetime() object or None if the timer is not started.
-        """
-        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
-        if ts:
-            return datetime.utcfromtimestamp(ts)
-
-    @property
-    def quoted_text(self):
-        """Text inside the quote
-
-        :returns: Quoted text"""
-        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
-
-    @property
-    def quote(self):
-        """Quote getter
-
-        :returns: Quoted message, if found in the database"""
-        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
-        if msg:
-            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
-
-    @quote.setter
-    def quote(self, quoted_message):
-        """Quote setter"""
-        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
-
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -238,11 +166,6 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
-    @property
-    def error(self):
-        """Error message"""
-        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
@@ -253,20 +176,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self):
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -379,10 +288,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -403,22 +308,21 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    const.DC_MSG_TEXT: 'text',
+    const.DC_MSG_IMAGE: 'image',
+    const.DC_MSG_GIF: 'gif',
+    const.DC_MSG_AUDIO: 'audio',
+    const.DC_MSG_VIDEO: 'video',
+    const.DC_MSG_FILE: 'file'
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
-    if code is not None:
-        return code
+    for code, value in _view_type_mapping.items():
+        if value == view_type_name:
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
 
 
 #
@@ -428,43 +332,20 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if not res:
-            return
-        action, affected, actor = res
-        affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
-        return "ac_member_" + res[0], d
-
-
-def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
-    if m:
-        text = m.group(1)
-    text = text.rstrip(".")
-    return text.strip()
+        if res:
+            contact = msg.account.get_contact_by_addr(res[1])
+            if contact:
+                d = dict(chat=msg.chat, contact=contact, message=msg)
+                return "ac_member_" + res[0], d
 
 
 def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
-
-    returns a (action, affected, actor) triple """
-
     # Member Me (x@y) removed by a@b.
-    # Member x@y added by a@b
-    # Member With space (tmp1@x.org) removed by tmp2@x.org.
-    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-    # Group left by some one (tmp1@x.org).
-    # Group left by tmp1@x.org.
+    # Member x@y removed by a@b
     text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
-    if m:
-        affected, action, actor = m.groups()
-        return action, extract_addr(affected), extract_addr(actor)
-    if text.startswith("group left by "):
-        addr = extract_addr(text[13:])
-        if addr:
-            return "removed", addr, addr
+    parts = text.split()
+    if parts[0] == "member":
+        if parts[2] in ("removed", "added"):
+            return parts[2], parts[1]
+        if parts[3] in ("removed", "added"):
+            return parts[3], parts[2].strip("()")

python/src/deltachat/testplugin.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 import os
 import sys
-import io
 import subprocess
 import queue
 import threading
@@ -17,9 +16,6 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
-from deltachat import direct_imap
-from deltachat.account import parseaddr
-
 
 import deltachat
 
@@ -34,16 +30,12 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    parser.addoption(
-        "--strict-tls", action="store_true",
-        help="Never accept invalid TLS certificates for test accounts",
-    )
-    parser.addoption(
-        "--provider-file", "-P", default=None,
-        help="file which contains config settings for real-world providers")
 
 
 def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -130,30 +122,17 @@ def pytest_report_header(config, startdir):
     return summary
 
 
-def parse_accountfile(fn):
-    if fn is None:
-        return []
-
-    for line in open(fn):
-        if line.strip() and not line.strip().startswith('#'):
-            config_dict = {}
-            for part in line.split():
-                name, value = part.split("=")
-                config_dict[name] = value
-            yield config_dict
-
-
-def pytest_generate_tests(metafunc):
-    if "real_provider_config" in metafunc.fixturenames:
-        account_configs = list(parse_accountfile(metafunc.config.getoption("--provider-file")))
-        ids = [parseaddr(cfg["addr"])[1] for cfg in account_configs]
-        metafunc.parametrize("real_provider_config", account_configs, ids=ids)
-
-
 class SessionLiveConfigFromFile:
     def __init__(self, fn):
         self.fn = fn
-        self.configlist = list(parse_accountfile(fn))
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
 
     def get(self, index):
         return self.configlist[index]
@@ -174,7 +153,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -237,7 +216,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -248,27 +226,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
-
-        def make_account_from_real_config(self, provider_config):
-            configdict = provider_config
-            addr = parseaddr(configdict["addr"])[1]
-            domain = addr.split("@")[1]
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-            tmpdb = tmpdir.join(domain)
-            return self.make_account(tmpdb.strpath, logid=domain)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
             if not quiet:
-                logger = FFIEventLogger(ac)
-                logger.init_time = self.init_time
-                ac.add_account_plugin(logger)
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
             self._accounts.append(ac)
             return ac
 
@@ -278,7 +242,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -313,15 +280,16 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            if pytestconfig.getoption("--strict-tls"):
-                # Enable strict certificate checks for online accounts
-                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -333,42 +301,41 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac._configtracker = ac.configure()
+            ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io([ac1])
+            ac1.wait_configure_finish()
+            ac1.start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
+            ac1.wait_configure_finish()
+            ac1.start_io()
+            ac2.wait_configure_finish()
+            ac2.start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+        def get_many_online_accounts(self, num, move=True, quiet=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
                         for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
             for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
+                acc._configtracker.wait_finish()
+                acc.start_io()
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
-
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -376,51 +343,14 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            if hasattr(account, "direct_imap"):
-                # Attach the existing direct_imap. If we did not do this, a new one would be created and
-                # delete existing messages (see dc_account_extra_configure(configure))
-                ac.direct_imap = account.direct_imap
-            ac._configtracker = ac.configure()
+            ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
-            started_accounts = []
-            for acc in accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    acc._evtracker.consume_events()
-                    acc.get_device_chat().mark_noticed()
-                    del acc._configtracker
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
-
-            for acc in started_accounts:
-                acc._evtracker.wait_all_initial_fetches()
-
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
-
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
             args = [
                 sys.executable,
                 "-u",
@@ -445,42 +375,9 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1, ac2):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
+    return am
 
 
 class BotProcess:
@@ -548,20 +445,4 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
-
-        def indent(self, msg):
-            print("  " + msg)
-
     return Printer()
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    # execute all other hooks to obtain the report object
-    outcome = yield
-    rep = outcome.get_result()
-
-    # set a report attribute for each phase of a call, which can
-    # be "setup", "call", "teardown"
-
-    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl, Global
+from .hookspec import account_hookimpl
 
 
 class ImexFailed(RuntimeError):
@@ -20,16 +20,6 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
-
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []
@@ -50,14 +40,12 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self, account):
-        self.account = account
+    def __init__(self):
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
-        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -71,10 +59,7 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
-        if success:
-            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
-        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/real_providers.py

@@ -1,72 +0,0 @@
-import pytest
-
-
-@pytest.fixture
-def acprovider(acfactory, real_provider_config):
-    ac = acfactory.make_account_from_real_config(real_provider_config)
-    ac.update_config(real_provider_config)
-    ac._configtracker = ac.configure()
-    return ac
-
-
-@pytest.fixture
-def actest(acfactory):
-    return acfactory.get_online_configuring_account()
-
-
-def test_configure_success(acfactory, acprovider, lp):
-    lp.sec("waiting for successful configuration of provider account")
-    acfactory.wait_configure_and_start_io()
-
-    assert acprovider.is_configured()
-    for name in ("inbox", "mvbox", "sentbox"):
-        folder = acprovider.get_config("configured_" + name + "_folder")
-        if not folder:
-            lp.sec("found no {} folder".format(name))
-            continue
-
-        lp.sec("removing provider account IMAP folder {}".format(folder))
-        acprovider.direct_imap.select_folder(folder)
-        acprovider.direct_imap.delete("1:*")
-
-
-def test_basic_send_receive(acprovider, actest, acfactory, lp):
-    acfactory.wait_configure_and_start_io()
-
-    lp.sec("sending message from test account to provider account")
-    chat = actest.create_chat(acprovider)
-    chat.send_text("hello")
-
-    lp.sec("receiving message with the provider account")
-    msg = acprovider._evtracker.wait_next_messages_changed()
-    assert msg.chat.is_deaddrop() and not msg.is_encrypted()
-
-    lp.sec("sending message back from provider to test account")
-    back_chat = acprovider.create_chat(actest)
-    back_chat.send_text("world")
-
-    lp.sec("waiting with test account for provider mail")
-    msg = actest._evtracker.wait_next_incoming_message()
-    assert msg.text == "world"
-    assert msg.is_encrypted()
-
-
-def test_group_messages(acprovider, actest, acfactory, lp):
-    acfactory.wait_configure_and_start_io()
-
-    lp.sec("sending message from test account to provider account")
-    chat = actest.create_chat(acprovider)
-    chat.send_text("hello")
-
-    lp.sec("receiving message with the provider account")
-    msg = acprovider._evtracker.wait_next_messages_changed()
-    assert msg.chat.is_deaddrop() and not msg.is_encrypted()
-
-    lp.sec("sending message back from provider to test account")
-    back_chat = acprovider.create_chat(actest)
-    back_chat.send_text("world")
-
-    lp.sec("waiting with test account for provider mail")
-    msg = actest._evtracker.wait_next_incoming_message()
-    assert msg.text == "world"
-    assert msg.is_encrypted()

python/tests/stress_test_db.py

@@ -110,7 +110,7 @@ class AutoReplier:
         if self.current_sent >= self.num_send:
             self.report_func(self, ReportType.exit)
             return
-        message.create_chat()
+        message.accept_sender_contact()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 

python/tests/test_increation.py

@@ -36,7 +36,9 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -46,7 +48,9 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -60,7 +64,9 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        chat = ac1.create_chat(ac2)
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -74,7 +80,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(ac2)
+        chat2.add_contact(c2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
+    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
+    chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -1,17 +1,19 @@
 [tox]
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py37
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
+    DCC_PY_LIVECONFIG
     DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
@@ -45,9 +47,8 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    # Pin dependencies to the versions which actually work with Python 3.5.
-    sphinx==3.4.3
-    breathe==4.28.0
+    sphinx
+    breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 
@@ -70,8 +71,6 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
-markers = 
-    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   scripts/run-integration-tests.sh -e py35
+#   ./run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   scripts/run-integration-tests.sh -e py35 -- -x
+#   ./run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,6 +23,9 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
+    export DCC_PY_LIVECONFIG=liveconfig
+fi
 tox "$@"
 ret=$?
 popd

rust-toolchain

@@ -1 +1 @@
-1.50.0
+1.43.1

scripts/ci_upload.sh

@@ -1,75 +0,0 @@
-#!/bin/bash
-
-if [ -z "$DEVPI_LOGIN" ] ; then 
-    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
-    exit 0
-fi
-
-set -xe
-
-PYDOCDIR=${1:?directory with python docs}
-WHEELHOUSEDIR=${2:?directory with pre-built wheels}
-DOXYDOCDIR=${3:?directory where doxygen docs to be found}
-SSHTARGET=ci@b1.delta.chat
-
-
-# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
-export BRANCH=${CIRCLE_BRANCH:master}
-
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
-
-
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
-
-# C docs to c.delta.chat
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$DOXYDOCDIR/html/" \
-  delta@c.delta.chat:build-c/${BRANCH}
-
-echo -----------------------
-echo upload wheels 
-echo -----------------------
-
-# Bundle external shared libraries into the wheels
-
-ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
-scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  $WHEELHOUSEDIR \
-  $SSHTARGET:$BUILDDIR 
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-
-    # we rely on the "venv" virtualenv on the remote account to exist 
-    source venv/bin/activate
-    cd $BUILDDIR
-
-    devpi use https://m.devpi.net
-    devpi login dc --password $DEVPI_LOGIN
-
-    N_BRANCH=${BRANCH//[\/]}
-
-    devpi use dc/\$N_BRANCH || {
-        devpi index -c \$N_BRANCH 
-        devpi use dc/\$N_BRANCH
-    }
-    devpi index \$N_BRANCH bases=/root/pypi
-    devpi upload wheelhouse/deltachat*
-
-    # remove devpi non-master dc indices if thy are too old
-    # this script was copied above 
-    python cleanup_devpi_indices.py
-_HERE

scripts/coverage.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-set -eu
-
-if ! which grcov 2>/dev/null 1>&2; then
-    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
-    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
-    exit 1
-fi
-
-# Allow `-Z` flags without using nightly Rust.
-export RUSTC_BOOTSTRAP=1
-
-# We are using `-Zprofile` instead of source-based coverage [1]
-# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
-#
-# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
-# [2] https://github.com/mozilla/grcov/issues/595
-
-export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
-export RUSTDOCFLAGS="-Cpanic=abort"
-cargo clean
-cargo build
-cargo test
-
-grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/manual_remote_tests.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -xe
-
-JOB=${1:?need to specify 'rust' or 'python'}
-BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
-REPONAME="$(basename $(git rev-parse --show-toplevel))"
-
-time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"

scripts/remote_tests_rust.sh

@@ -1,29 +0,0 @@
-#!/bin/bash 
-
-BUILD_ID=${1:?specify build ID}
-
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
-
-set -e
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-echo "--- Running Rust tests remotely"
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-    export RUSTC_WRAPPER=\`which sccache\`
-    cd $BUILDDIR
-    export TARGET=x86_64-unknown-linux-gnu
-    export RUSTC_WRAPPER=sccache
-
-    bash scripts/run-rust-test.sh
-_HERE
-

scripts/run-doxygen.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-cd deltachat-ffi 
-PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

set_core_version.py

@@ -1,31 +1,22 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 import os
 import sys
 import re
 import pathlib
 import subprocess
-from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-
-def regex_matches(relpath, regex=rex):
+def read_toml_version(relpath):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = regex.match(line)
+        m = rex.match(line)
         if m is not None:
-            return m
-
-
-def read_toml_version(relpath):
-    res = regex_matches(relpath, rex)
-    if res is not None:
-        return res.group(1)
+            return m.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
-
 def replace_toml_version(relpath, newversion):
     p = pathlib.Path(relpath)
     assert p.exists()
@@ -34,28 +25,18 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
-                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
                 f.write(line)
     os.rename(tmp_path, str(p))
 
+if __name__ == "__main__":
 
-def main():
-    parser = ArgumentParser(prog="set_core_version")
-    parser.add_argument("newversion")
-
-    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
-    try:
-        opts = parser.parse_args()
-    except SystemExit:
-        print()
-        for x in toml_list:
+    if len(sys.argv) < 2:
+        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
             print("{}: {}".format(x, read_toml_version(x)))
-        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-
-    newversion = opts.newversion
+    newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -63,32 +44,22 @@ def main():
     ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
-    if "alpha" not in newversion:
-        for line in open("CHANGELOG.md"):
-            ## 1.25.0
-            if line.startswith("## "):
-                if line[2:].strip().startswith(newversion):
-                    break
-        else:
-            raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+    for line in open("CHANGELOG.md"):
+        ## 1.25.0
+        if line.startswith("## "):
+            if line[2:].strip().startswith(newversion):
+                break
+    else:
+        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
-    print("running cargo check")
     subprocess.call(["cargo", "check"])
-
-    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit, on master make sure to: ")
+    print("after commit make sure to: ")
     print("")
-    print("   git tag -a {}".format(newversion))
-    print("   git push origin {}".format(newversion))
+    print("   git tag {}".format(newversion))
     print("")
-
-
-if __name__ == "__main__":
-    main()
-

spec.md

@@ -1,12 +1,10 @@
-# chat-mail specification
+# Chat-over-Email specification
 
-Version: 0.32.0
-Status:  In-progress 
-Format:  [Semantic Line Breaks](https://sembr.org/)
+Version 0.30.0
 
-This document roughly describes how chat-mail 
-apps use the standard e-mail system 
-to implement typical messenger functions.
+This document describes how emails can be used
+to implement typical messenger functions
+while staying compatible to existing MUAs.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -32,14 +30,17 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
+If Memoryhole is not used,
+the subject of encrypted messages SHOULD be replaced by the string `...`.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD be `Message from <sender name>`.
+the `Subject` header SHOULD start with the characters `Chat:`
+and SHOULD be an excerpt of the message.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -57,7 +58,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Message from sender@domain
+    Subject: Chat: Hello ...
 
     Hello world!
 

src/accounts.rs

@@ -1,540 +0,0 @@
-use std::collections::BTreeMap;
-
-use async_std::fs;
-use async_std::path::PathBuf;
-use async_std::prelude::*;
-use async_std::sync::{Arc, RwLock};
-use uuid::Uuid;
-
-use anyhow::{ensure, Context as _, Result};
-use serde::{Deserialize, Serialize};
-
-use crate::context::Context;
-use crate::events::Event;
-
-/// Account manager, that can handle multiple accounts in a single place.
-#[derive(Debug, Clone)]
-pub struct Accounts {
-    dir: PathBuf,
-    config: Config,
-    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
-}
-
-impl Accounts {
-    /// Loads or creates an accounts folder at the given `dir`.
-    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
-        if !dir.exists().await {
-            Accounts::create(os_name, &dir).await?;
-        }
-
-        Accounts::open(dir).await
-    }
-
-    /// Creates a new default structure, including a default account.
-    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
-        fs::create_dir_all(dir)
-            .await
-            .context("failed to create folder")?;
-
-        // create default account
-        let config = Config::new(os_name.clone(), dir).await?;
-        let account_config = config.new_account(dir).await?;
-
-        Context::new(os_name, account_config.dbfile().into(), account_config.id)
-            .await
-            .context("failed to create default account")?;
-
-        Ok(())
-    }
-
-    /// Opens an existing accounts structure. Will error if the folder doesn't exist,
-    /// no account exists and no config exists.
-    pub async fn open(dir: PathBuf) -> Result<Self> {
-        ensure!(dir.exists().await, "directory does not exist");
-
-        let config_file = dir.join(CONFIG_NAME);
-        ensure!(config_file.exists().await, "accounts.toml does not exist");
-
-        let config = Config::from_file(config_file).await?;
-        let accounts = config.load_accounts().await?;
-
-        Ok(Self {
-            dir,
-            config,
-            accounts: Arc::new(RwLock::new(accounts)),
-        })
-    }
-
-    /// Get an account by its `id`:
-    pub async fn get_account(&self, id: u32) -> Option<Context> {
-        self.accounts.read().await.get(&id).cloned()
-    }
-
-    /// Get the currently selected account.
-    pub async fn get_selected_account(&self) -> Context {
-        let id = self.config.get_selected_account().await;
-        self.accounts
-            .read()
-            .await
-            .get(&id)
-            .cloned()
-            .expect("inconsistent state")
-    }
-
-    /// Select the given account.
-    pub async fn select_account(&self, id: u32) -> Result<()> {
-        self.config.select_account(id).await?;
-
-        Ok(())
-    }
-
-    /// Add a new account.
-    pub async fn add_account(&self) -> Result<u32> {
-        let os_name = self.config.os_name().await;
-        let account_config = self.config.new_account(&self.dir).await?;
-
-        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
-        self.accounts.write().await.insert(account_config.id, ctx);
-
-        Ok(account_config.id)
-    }
-
-    /// Remove an account.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
-        let ctx = self.accounts.write().await.remove(&id);
-        ensure!(ctx.is_some(), "no account with this id: {}", id);
-        let ctx = ctx.unwrap();
-        ctx.stop_io().await;
-        drop(ctx);
-
-        if let Some(cfg) = self.config.get_account(id).await {
-            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                .await
-                .context("failed to remove account data")?;
-        }
-        self.config.remove_account(id).await?;
-
-        Ok(())
-    }
-
-    /// Migrate an existing account into this structure.
-    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
-        let blobdir = Context::derive_blobdir(&dbfile);
-
-        ensure!(
-            dbfile.exists().await,
-            "no database found: {}",
-            dbfile.display()
-        );
-        ensure!(
-            blobdir.exists().await,
-            "no blobdir found: {}",
-            blobdir.display()
-        );
-
-        let old_id = self.config.get_selected_account().await;
-
-        // create new account
-        let account_config = self
-            .config
-            .new_account(&self.dir)
-            .await
-            .context("failed to create new account")?;
-
-        let new_dbfile = account_config.dbfile().into();
-        let new_blobdir = Context::derive_blobdir(&new_dbfile);
-
-        let res = {
-            fs::create_dir_all(&account_config.dir)
-                .await
-                .context("failed to create dir")?;
-            fs::rename(&dbfile, &new_dbfile)
-                .await
-                .context("failed to rename dbfile")?;
-            fs::rename(&blobdir, &new_blobdir)
-                .await
-                .context("failed to rename blobdir")?;
-            Ok(())
-        };
-
-        match res {
-            Ok(_) => {
-                let ctx = Context::with_blobdir(
-                    self.config.os_name().await,
-                    new_dbfile,
-                    new_blobdir,
-                    account_config.id,
-                )
-                .await?;
-                self.accounts.write().await.insert(account_config.id, ctx);
-                Ok(account_config.id)
-            }
-            Err(err) => {
-                // remove temp account
-                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
-                    .await
-                    .context("failed to remove account data")?;
-
-                self.config.remove_account(account_config.id).await?;
-
-                // set selection back
-                self.select_account(old_id).await?;
-
-                Err(err)
-            }
-        }
-    }
-
-    /// Get a list of all account ids.
-    pub async fn get_all(&self) -> Vec<u32> {
-        self.accounts.read().await.keys().copied().collect()
-    }
-
-    /// Import a backup using a new account and selects it.
-    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
-        let old_id = self.config.get_selected_account().await;
-
-        let id = self.add_account().await?;
-        let ctx = self.get_account(id).await.expect("just added");
-
-        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
-            Ok(_) => Ok(id),
-            Err(err) => {
-                // remove temp account
-                self.remove_account(id).await?;
-                // set selection back
-                self.select_account(old_id).await?;
-                Err(err)
-            }
-        }
-    }
-
-    pub async fn start_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
-            account.start_io().await;
-        }
-    }
-
-    pub async fn stop_io(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
-            account.stop_io().await;
-        }
-    }
-
-    pub async fn maybe_network(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
-            account.maybe_network().await;
-        }
-    }
-
-    /// Unified event emitter.
-    pub async fn get_event_emitter(&self) -> EventEmitter {
-        let emitters: Vec<_> = self
-            .accounts
-            .read()
-            .await
-            .iter()
-            .map(|(_id, a)| a.get_event_emitter())
-            .collect();
-
-        EventEmitter(futures::stream::select_all(emitters))
-    }
-}
-
-#[derive(Debug)]
-pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
-
-impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
-    pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv())
-    }
-
-    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
-    pub async fn recv(&mut self) -> Option<Event> {
-        self.0.next().await
-    }
-}
-
-impl async_std::stream::Stream for EventEmitter {
-    type Item = Event;
-
-    fn poll_next(
-        mut self: std::pin::Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self.0).poll_next(cx)
-    }
-}
-
-pub const CONFIG_NAME: &str = "accounts.toml";
-pub const DB_NAME: &str = "dc.db";
-
-#[derive(Debug, Clone)]
-pub struct Config {
-    file: PathBuf,
-    inner: Arc<RwLock<InnerConfig>>,
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-struct InnerConfig {
-    pub os_name: String,
-    /// The currently selected account.
-    pub selected_account: u32,
-    pub next_id: u32,
-    pub accounts: Vec<AccountConfig>,
-}
-
-impl Config {
-    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
-        let cfg = Config {
-            file: dir.join(CONFIG_NAME),
-            inner: Arc::new(RwLock::new(InnerConfig {
-                os_name,
-                accounts: Vec::new(),
-                selected_account: 0,
-                next_id: 1,
-            })),
-        };
-
-        cfg.sync().await?;
-
-        Ok(cfg)
-    }
-
-    pub async fn os_name(&self) -> String {
-        self.inner.read().await.os_name.clone()
-    }
-
-    /// Sync the inmemory representation to disk.
-    async fn sync(&self) -> Result<()> {
-        fs::write(
-            &self.file,
-            toml::to_string_pretty(&*self.inner.read().await)?,
-        )
-        .await
-        .context("failed to write config")
-    }
-
-    /// Read a configuration from the given file into memory.
-    pub async fn from_file(file: PathBuf) -> Result<Self> {
-        let bytes = fs::read(&file).await.context("failed to read file")?;
-        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
-
-        Ok(Config {
-            file,
-            inner: Arc::new(RwLock::new(inner)),
-        })
-    }
-
-    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
-        let cfg = &*self.inner.read().await;
-        let mut accounts = BTreeMap::new();
-        for account_config in &cfg.accounts {
-            let ctx = Context::new(
-                cfg.os_name.clone(),
-                account_config.dbfile().into(),
-                account_config.id,
-            )
-            .await?;
-            accounts.insert(account_config.id, ctx);
-        }
-
-        Ok(accounts)
-    }
-
-    /// Create a new account in the given root directory.
-    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
-        let id = {
-            let inner = &mut self.inner.write().await;
-            let id = inner.next_id;
-            let uuid = Uuid::new_v4();
-            let target_dir = dir.join(uuid.to_simple_ref().to_string());
-
-            inner.accounts.push(AccountConfig {
-                id,
-                dir: target_dir.into(),
-                uuid,
-            });
-            inner.next_id += 1;
-            id
-        };
-
-        self.sync().await?;
-
-        self.select_account(id).await.expect("just added");
-        let cfg = self.get_account(id).await.expect("just added");
-        Ok(cfg)
-    }
-
-    /// Removes an existing acccount entirely.
-    pub async fn remove_account(&self, id: u32) -> Result<()> {
-        {
-            let inner = &mut *self.inner.write().await;
-            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
-                // remove account from the configs
-                inner.accounts.remove(idx);
-            }
-            if inner.selected_account == id {
-                // reset selected account
-                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
-            }
-        }
-
-        self.sync().await
-    }
-
-    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
-        self.inner
-            .read()
-            .await
-            .accounts
-            .iter()
-            .find(|e| e.id == id)
-            .cloned()
-    }
-
-    pub async fn get_selected_account(&self) -> u32 {
-        self.inner.read().await.selected_account
-    }
-
-    pub async fn select_account(&self, id: u32) -> Result<()> {
-        {
-            let inner = &mut *self.inner.write().await;
-            ensure!(
-                inner.accounts.iter().any(|e| e.id == id),
-                "invalid account id: {}",
-                id
-            );
-
-            inner.selected_account = id;
-        }
-
-        self.sync().await?;
-        Ok(())
-    }
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub struct AccountConfig {
-    /// Unique id.
-    pub id: u32,
-    /// Root directory for all data for this account.
-    pub dir: std::path::PathBuf,
-    pub uuid: Uuid,
-}
-
-impl AccountConfig {
-    /// Get the canoncial dbfile name for this configuration.
-    pub fn dbfile(&self) -> std::path::PathBuf {
-        self.dir.join(DB_NAME)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[async_std::test]
-    async fn test_account_new_open() {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts1").into();
-
-        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        let accounts2 = Accounts::open(p).await.unwrap();
-
-        assert_eq!(accounts1.accounts.read().await.len(), 1);
-        assert_eq!(accounts1.config.get_selected_account().await, 1);
-
-        assert_eq!(accounts1.dir, accounts2.dir);
-        assert_eq!(
-            &*accounts1.config.inner.read().await,
-            &*accounts2.config.inner.read().await,
-        );
-        assert_eq!(
-            accounts1.accounts.read().await.len(),
-            accounts2.accounts.read().await.len()
-        );
-    }
-
-    #[async_std::test]
-    async fn test_account_new_add_remove() {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
-
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, 2);
-        assert_eq!(accounts.config.get_selected_account().await, id);
-        assert_eq!(accounts.accounts.read().await.len(), 2);
-
-        accounts.select_account(1).await.unwrap();
-        assert_eq!(accounts.config.get_selected_account().await, 1);
-
-        accounts.remove_account(1).await.unwrap();
-        assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-    }
-
-    #[async_std::test]
-    async fn test_migrate_account() {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
-
-        let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
-            .await
-            .unwrap();
-        ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
-            .await
-            .unwrap();
-
-        drop(ctx);
-
-        accounts
-            .migrate_account(extern_dbfile.clone())
-            .await
-            .unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 2);
-        assert_eq!(accounts.config.get_selected_account().await, 2);
-
-        let ctx = accounts.get_selected_account().await;
-        assert_eq!(
-            "me@mail.com",
-            ctx.get_config(crate::config::Config::Addr)
-                .await
-                .unwrap()
-                .unwrap()
-        );
-    }
-
-    /// Tests that accounts are sorted by ID.
-    #[async_std::test]
-    async fn test_accounts_sorted() {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-
-        for expected_id in 2..10 {
-            let id = accounts.add_account().await.unwrap();
-            assert_eq!(id, expected_id);
-        }
-
-        let ids = accounts.get_all().await;
-        for (i, expected_id) in (1..10).enumerate() {
-            assert_eq!(ids.get(i), Some(&expected_id));
-        }
-    }
-}

src/aheader.rs

@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::addr_cmp;
+use crate::contact::*;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -1,6 +1,5 @@
 //! # Blob directory management
 
-use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 
@@ -8,23 +7,12 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::format_err;
-use anyhow::Context as _;
-use anyhow::Error;
-use image::DynamicImage;
 use image::GenericImageView;
-use image::ImageFormat;
-use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::config::Config;
-use crate::constants::{
-    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
-    WORSE_IMAGE_SIZE,
-};
+use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
-use crate::events::EventType;
-use crate::message;
+use crate::events::Event;
 
 /// Represents a file in the blob directory.
 ///
@@ -58,30 +46,24 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: &str,
+        suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err.into(),
+                cause: err,
             })?;
-
-        // workaround a bug in async-std
-        // (the executor does not handle blocking operation in Drop correctly,
-        // see https://github.com/async-rs/async-std/issues/900 )
-        let _ = file.flush().await;
-
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -137,17 +119,18 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: &Path,
+        src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file = fs::File::open(src)
-            .await
-            .map_err(|err| BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: String::from(""),
-                src: src.to_path_buf(),
-                cause: err,
-            })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
+        let mut src_file =
+            fs::File::open(src.as_ref())
+                .await
+                .map_err(|err| BlobError::CopyFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: String::from(""),
+                    src: src.as_ref().to_path_buf(),
+                    cause: err,
+                })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
@@ -160,19 +143,15 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.to_path_buf(),
+                src: src.as_ref().to_path_buf(),
                 cause: err,
             });
         }
-
-        // workaround, see create() for details
-        let _ = dst_file.flush().await;
-
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -185,22 +164,17 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
-    /// Paths into the blob directory may be either defined by an absolute path
-    /// or by the relative prefix `$BLOBDIR`.
-    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        if src.starts_with(context.get_blobdir()) {
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'_>, BlobError> {
+        if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.starts_with("$BLOBDIR/") {
-            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -221,22 +195,23 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &'a Context,
-        path: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let rel_path =
-            path.strip_prefix(context.get_blobdir())
-                .map_err(|_| BlobError::WrongBlobdir {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    src: path.to_path_buf(),
-                })?;
-        if !BlobObject::is_acceptible_blob_name(rel_path) {
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::WrongBlobdir {
+                blobdir: context.get_blobdir().to_path_buf(),
+                src: path.as_ref().to_path_buf(),
+            })?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.to_path_buf(),
+                blobname: path.as_ref().to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.to_path_buf(),
+            blobname: path.as_ref().to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -380,188 +355,27 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
+    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
-
-        let img_wh =
-            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
-                .unwrap_or_default()
-            {
-                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
-                MediaQuality::Worse => WORSE_AVATAR_SIZE,
-            };
-
-        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
-        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
-        if let Some(new_name) = self
-            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
-            .await?
-        {
-            self.name = new_name;
-        }
-        Ok(())
-    }
-
-    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
-        let blob_abs = self.to_abs_path();
-        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
-            != Some((Viewtype::Image, "image/jpeg"))
-        {
-            return Ok(());
-        }
-
-        let img_wh =
-            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
-                .unwrap_or_default()
-            {
-                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
-                MediaQuality::Worse => WORSE_IMAGE_SIZE,
-            };
-
-        if self
-            .recode_to_size(context, blob_abs, img_wh, None)
-            .await?
-            .is_some()
-        {
-            return Err(format_err!(
-                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
-            )
-            .into());
-        }
-        Ok(())
-    }
-
-    async fn recode_to_size(
-        &self,
-        context: &Context,
-        mut blob_abs: PathBuf,
-        mut img_wh: u32,
-        max_bytes: Option<usize>,
-    ) -> Result<Option<String>, BlobError> {
-        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
-        let orientation = self.get_exif_orientation(context);
-        let mut encoded = Vec::new();
-        let mut changed_name = None;
 
-        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
-            encoded.clear();
-            img.write_to(encoded, image::ImageFormat::Jpeg)?;
-            Ok(())
-        }
-        fn encode_img_exceeds_bytes(
-            context: &Context,
-            img: &DynamicImage,
-            max_bytes: Option<usize>,
-            encoded: &mut Vec<u8>,
-        ) -> anyhow::Result<bool> {
-            if let Some(max_bytes) = max_bytes {
-                encode_img(img, encoded)?;
-                if encoded.len() > max_bytes {
-                    info!(
-                        context,
-                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
-                        encoded.len(),
-                        img.width(),
-                        img.height(),
-                        max_bytes,
-                    );
-                    return Ok(true);
-                }
-            }
-            Ok(false)
-        }
-        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
-
-        let do_scale =
-            exceeds_width || encode_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
-        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
-
-        if do_scale || do_rotate {
-            if do_rotate {
-                img = match orientation {
-                    Ok(90) => img.rotate90(),
-                    Ok(180) => img.rotate180(),
-                    Ok(270) => img.rotate270(),
-                    _ => img,
-                }
-            }
-
-            if do_scale {
-                if !exceeds_width {
-                    // The image is already smaller than img_wh, but exceeds max_bytes
-                    // We can directly start with trying to scale down to 2/3 of its current width
-                    img_wh = max(img.width(), img.height()) * 2 / 3
-                }
-
-                loop {
-                    let new_img = img.thumbnail(img_wh, img_wh);
-
-                    if encode_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
-                        if img_wh < 20 {
-                            return Err(format_err!(
-                                "Failed to scale image to below {}B",
-                                max_bytes.unwrap_or_default()
-                            )
-                            .into());
-                        }
-
-                        img_wh = img_wh * 2 / 3;
-                    } else {
-                        info!(
-                            context,
-                            "Final scaled-down image size: {}B ({}px)",
-                            encoded.len(),
-                            img_wh
-                        );
-                        break;
-                    }
-                }
-            }
-
-            // The file format is JPEG now, we may have to change the file extension
-            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
-                blob_abs = blob_abs.with_extension("jpg");
-                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
-                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
-                changed_name = Some(format!("$BLOBDIR/{}", file_name));
-            }
-
-            if encoded.is_empty() {
-                encode_img(&img, &mut encoded)?;
-            }
-
-            fs::write(&blob_abs, &encoded)
-                .await
-                .map_err(|err| BlobError::WriteFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                    cause: err.into(),
-                })?;
+        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
+            return Ok(());
         }
 
-        Ok(changed_name)
-    }
+        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
 
-    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
-        let file = std::fs::File::open(self.to_abs_path())?;
-        let mut bufreader = std::io::BufReader::new(&file);
-        let exifreader = exif::Reader::new();
-        let exif = exifreader.read_from_container(&mut bufreader)?;
-        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
-            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
-            // we only use rotation, in practise, flipping is not used.
-            match orientation.value.get_uint(0) {
-                Some(3) => return Ok(180),
-                Some(6) => return Ok(90),
-                Some(8) => return Ok(270),
-                other => warn!(context, "exif orientation value ignored: {:?}", other),
-            }
-        }
-        Ok(0)
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        Ok(())
     }
 }
 
@@ -586,7 +400,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: anyhow::Error,
+        cause: std::io::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -607,67 +421,75 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
-    use fs::File;
-
     use super::*;
 
-    use crate::test_utils::TestContext;
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_create() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
-        let fname = t.get_blobdir().join("foo");
+        let t = dummy_context().await;
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
+        let t = dummy_context().await;
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let t = dummy_context().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let t = dummy_context().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let t = dummy_context().await;
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
-        let t = TestContext::new().await;
-        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
-        let foo_path = t.get_blobdir().join("foo.txt");
+        let t = dummy_context().await;
+        BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
+        let foo_path = t.ctx.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
-        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+        BlobObject::create(&t.ctx, "foo.txt", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -682,16 +504,16 @@ mod tests {
 
     #[async_std::test]
     async fn test_double_ext_preserved() {
-        let t = TestContext::new().await;
-        BlobObject::create(&t, "foo.tar.gz", b"hello")
+        let t = dummy_context().await;
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t, "foo.tar.gz", b"world")
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -707,59 +529,53 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_long_names() {
-        let t = TestContext::new().await;
+        let t = dummy_context().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
 
     #[async_std::test]
     async fn test_create_and_copy() {
-        let t = TestContext::new().await;
+        let t = dummy_context().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
-            .await
-            .is_err());
-        let whoops = t.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
     #[async_std::test]
     async fn test_create_from_path() {
-        let t = TestContext::new().await;
+        let t = dummy_context().await;
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.get_blobdir().join("internal");
+        let src_int = t.ctx.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
     }
     #[async_std::test]
     async fn test_create_from_name_long() {
-        let t = TestContext::new().await;
+        let t = dummy_context().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -778,141 +594,8 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (stem, ext) =
+        let (_, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
-        assert!(!stem.is_empty());
-
-        // the extensions are kept together as between stem and extension a number may be added -
-        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
-        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
-        assert_eq!(stem, "wot");
-        assert_eq!(ext, ".tar.gz");
-
-        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
-        assert_eq!(stem, "");
-        assert_eq!(ext, ".foo.bar");
-
-        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
-        assert!(stem.contains("foo"));
-        assert!(!stem.contains('?'));
-        assert_eq!(ext, ".bar");
-
-        let (stem, ext) = BlobObject::sanitise_name("no-extension");
-        assert_eq!(stem, "no-extension");
-        assert_eq!(ext, "");
-
-        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
-        assert_eq!(ext, ".c");
-        assert!(!stem.contains("path"));
-        assert!(!stem.contains("ignored"));
-        assert!(stem.contains("this"));
-        assert!(stem.contains("forbidden"));
-        assert!(!stem.contains('/'));
-        assert!(!stem.contains('\\'));
-        assert!(!stem.contains(':'));
-        assert!(!stem.contains('*'));
-        assert!(!stem.contains('?'));
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(&avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-
-        async fn file_size(path_buf: &PathBuf) -> u64 {
-            let file = File::open(path_buf).await.unwrap();
-            file.metadata().await.unwrap().len()
-        }
-
-        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
-
-        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
-            .await
-            .unwrap();
-        assert!(file_size(&avatar_blob).await <= 3000);
-        assert!(file_size(&avatar_blob).await > 2000);
-        let img = image::open(&avatar_blob).unwrap();
-        assert!(img.width() > 130);
-        assert_eq!(img.width(), img.height());
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
-        assert_eq!(
-            avatar_cfg,
-            avatar_src.with_extension("jpg").to_str().unwrap()
-        );
-
-        let img = image::open(avatar_cfg).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
     }
 }

src/chatlist.rs

@@ -1,19 +1,14 @@
 //! # Chat list module
 
-use anyhow::{bail, ensure, Result};
-
-use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
-use crate::constants::{
-    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
-    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
-    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
-};
-use crate::contact::Contact;
-use crate::context::Context;
-use crate::ephemeral::delete_expired_messages;
+use crate::chat;
+use crate::chat::*;
+use crate::constants::*;
+use crate::contact::*;
+use crate::context::*;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock_str;
+use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -62,8 +57,9 @@ impl Chatlist {
     ///   messages from addresses that have no relationship to the configured account.
     ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
     ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
-    ///   and offers the options "Start chat", "Block" and "Not now";
-    ///   The decision should be passed to dc_decide_on_contact_request().
+    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
+    ///   or "Not now".
+    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -80,7 +76,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -103,7 +99,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_expired_messages(context).await {
+        if let Err(err) = delete_device_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -121,9 +117,10 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
-                .await?
+            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .await
                 .unwrap_or_default()
+                .0
         } else {
             ChatId::new(0)
         };
@@ -150,12 +147,11 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
@@ -177,12 +173,11 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?))
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
@@ -211,12 +206,11 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
@@ -230,9 +224,10 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
-                    .await?
+                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .await
                     .unwrap_or_default()
+                    .0
             } else {
                 ChatId::new(0)
             };
@@ -241,12 +236,11 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
@@ -257,11 +251,13 @@ impl Chatlist {
                 process_rows,
             ).await?;
             if !flag_no_specials {
-                if let Some(last_deaddrop_fresh_msg_id) =
-                    get_last_deaddrop_fresh_msg(context).await?
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
                 {
                     if !flag_for_forwarding {
-                        ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
+                        ids.insert(
+                            0,
+                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                        );
                     }
                 }
                 add_archived_link_item = true;
@@ -269,11 +265,11 @@ impl Chatlist {
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
+                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
+            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -328,59 +324,50 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
+        let mut ret = Lot::new();
+
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
             None => {
-                let mut ret = Lot::new();
                 ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return Lot::new();
+                return ret;
             }
         };
 
-        Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
-    }
-
-    pub async fn get_summary2(
-        context: &Context,
-        chat_id: ChatId,
-        lastmsg_id: MsgId,
-        chat: Option<&Chat>,
-    ) -> Lot {
-        let mut ret = Lot::new();
-
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
-        let (lastmsg, lastcontact) =
-            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
-                if lastmsg.from_id == DC_CONTACT_ID_SELF {
-                    (Some(lastmsg), None)
-                } else {
-                    match chat.typ {
-                        Chattype::Group | Chattype::Mailinglist => {
-                            let lastcontact =
-                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
-                            (Some(lastmsg), lastcontact)
-                        }
-                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
-                    }
-                }
-            } else {
-                (None, None)
-            };
+        let mut lastcontact = None;
+
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
+                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
+            {
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
+            }
+
+            Some(lastmsg)
+        } else {
+            None
+        };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(stock_str::no_messages(context).await);
+            ret.text2 = Some(
+                context
+                    .stock_str(StockMessage::NoMessages)
+                    .await
+                    .to_string(),
+            );
         } else {
             ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
                 .await;
@@ -395,23 +382,25 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
-    let count = context
+pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
+    context
         .sql
-        .count(
+        .query_get_value(
+            context,
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
             paramsv![],
         )
-        .await?;
-    Ok(count)
+        .await
+        .unwrap_or_default()
 }
 
-async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>> {
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
     // We have an index over the state-column, this should be
     // sufficient as there are typically only few fresh messages.
-    let id = context
+    context
         .sql
         .query_get_value(
+            context,
             concat!(
                 "SELECT m.id",
                 " FROM msgs m",
@@ -424,37 +413,30 @@ async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>>
             ),
             paramsv![],
         )
-        .await?;
-    Ok(id)
+        .await
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
-    use crate::constants::Viewtype;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::message;
-    use crate::message::ContactRequestDecision;
-    use crate::stock_str::StockMessage;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_try_load() {
-        let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        let t = dummy_context().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -463,24 +445,26 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t, ChatVisibility::Archived)
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -488,24 +472,24 @@ mod tests {
 
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
-        let t = TestContext::new().await;
-        t.update_device_chats().await.unwrap();
-        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        let t = dummy_context().await;
+        t.ctx.update_device_chats().await.unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -513,178 +497,50 @@ mod tests {
 
     #[async_std::test]
     async fn test_search_special_chat_names() {
-        let t = TestContext::new().await;
-        t.update_device_chats().await.unwrap();
+        let t = dummy_context().await;
+        t.ctx.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.ctx
+            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.ctx
+            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
-    async fn test_search_single_chat() -> anyhow::Result<()> {
-        let t = TestContext::new_alice().await;
-
-        // receive a one-to-one-message, accept contact request
-        dc_receive_imf(
-            &t,
-            b"From: Bob Authname <bob@example.org>\n\
-                 To: alice@example.com\n\
-                 Subject: foo\n\
-                 Message-ID: <msg1234@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
-                 \n\
-                 hello foo\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .await?;
-
-        let chats = Chatlist::try_load(&t, 0, Some("Bob Authname"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.get_chat_id(), DC_CHAT_ID_DEADDROP);
-
-        let chat_id =
-            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
-                .await
-                .unwrap();
-        let contacts = get_chat_contacts(&t, chat_id).await?;
-        let contact_id = *contacts.first().unwrap();
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Authname");
-
-        // check, the one-to-one-chat can be found using chatlist search query
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // change the name of the contact; this also changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Nickname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 1);
-
-        // revert contact to authname, this again changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Authname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
-        let t = TestContext::new_alice().await;
-
-        // receive a one-to-one-message without authname set, accept contact request
-        dc_receive_imf(
-            &t,
-            b"From: bob@example.org\n\
-                 To: alice@example.com\n\
-                 Subject: foo\n\
-                 Message-ID: <msg5678@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
-                 \n\
-                 hello foo\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .await?;
-
-        let msg = t.get_last_msg().await;
-        let chat_id =
-            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
-                .await
-                .unwrap();
-        let contacts = get_chat_contacts(&t, chat_id).await?;
-        let contact_id = *contacts.first().unwrap();
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "bob@example.org");
-
-        // check, the one-to-one-chat can be found using chatlist search query
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // change the name of the contact; this also changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Nickname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
-        let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // revert name change, this again changes the name of the one-to-one-chat to the email-address
-        let test_id = Contact::create(&t, "", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "bob@example.org");
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        // finally, also check that a simple substring-search is working with email-addresses
-        let chats = Chatlist::try_load(&t, 0, Some("b@exa"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("b@exac"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        Ok(())
-    }
-
     #[async_std::test]
     async fn test_get_summary_unwrap() {
-        let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        let t = dummy_context().await;
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
+        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t, 0, None).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -1,62 +0,0 @@
-//! Implementation of Consistent Color Generation
-//!
-//! Consistent Color Generation is defined in XEP-0392.
-//!
-//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
-//! corresponding settings.
-use hsluv::hsluv_to_rgb;
-use sha1::{Digest, Sha1};
-
-/// Converts an identifier to Hue angle.
-fn str_to_angle(s: &str) -> f64 {
-    let bytes = s.as_bytes();
-    let result = Sha1::digest(bytes);
-    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
-        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
-    f64::from(checksum) / 65536.0 * 360.0
-}
-
-/// Converts RGB tuple to a 24-bit number.
-///
-/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
-/// most significant bits corresponding to the red color.
-fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
-    let r = ((r * 256.0) as u32).min(255);
-    let g = ((g * 256.0) as u32).min(255);
-    let b = ((b * 256.0) as u32).min(255);
-    65536 * r + 256 * g + b
-}
-
-/// Converts an identifier to RGB color.
-///
-/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
-/// half (50.0) to make colors suitable both for light and dark theme.
-pub(crate) fn str_to_color(s: &str) -> u32 {
-    rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_str_to_angle() {
-        // Test against test vectors from
-        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
-        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
-        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
-        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
-        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
-        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
-    }
-
-    #[test]
-    fn test_rgb_to_u32() {
-        assert_eq!(rgb_to_u32((0.0, 0.0, 0.0)), 0);
-        assert_eq!(rgb_to_u32((1.0, 1.0, 1.0)), 0xffffff);
-        assert_eq!(rgb_to_u32((0.0, 0.0, 1.0)), 0x0000ff);
-        assert_eq!(rgb_to_u32((0.0, 1.0, 0.0)), 0x00ff00);
-        assert_eq!(rgb_to_u32((1.0, 0.0, 0.0)), 0xff0000);
-        assert_eq!(rgb_to_u32((1.0, 0.5, 0.0)), 0xff8000);
-    }
-}

src/config.rs

@@ -1,6 +1,5 @@
 //! # Key-value configuration management
 
-use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -8,13 +7,11 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
-use crate::events::EventType;
-use crate::job;
+use crate::dc_tools::*;
+use crate::events::Event;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::provider::{get_provider_by_id, Provider};
-use crate::stock_str;
+use crate::stock::StockMessage;
 
 /// The available configuration keys.
 #[derive(
@@ -27,16 +24,17 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
-    MailSecurity,
     ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
-    SendSecurity,
     SmtpCertificateChecks,
     ServerFlags,
 
+    #[strum(props(default = "INBOX"))]
+    ImapFolder,
+
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -62,21 +60,12 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
-    #[strum(props(default = "0"))]
-    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
-
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
-    /// If set to "1", on the first time `start_io()` is called after configuring,
-    /// the newest existing messages are fetched.
-    /// Existing recipients are added to the contact database regardless of this setting.
-    #[strum(props(default = "1"))]
-    FetchExistingMsgs,
-
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -115,12 +104,6 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
-    ConfiguredInboxFolder,
-    ConfiguredMvboxFolder,
-    ConfiguredSentboxFolder,
-    ConfiguredSpamFolder,
-    ConfiguredTimestamp,
-    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -131,185 +114,126 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
-
-    Bot,
-
-    /// Whether we send a warning if the password is wrong (set to false when we send a warning
-    /// because we do not want to send a second warning)
-    #[strum(props(default = "0"))]
-    NotifyAboutWrongPw,
-
-    /// address to webrtc instance to use for videochats
-    WebrtcInstance,
-
-    /// Timestamp of the last time housekeeping was run
-    LastHousekeeping,
-
-    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
-    #[strum(props(default = "60"))]
-    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
-    pub async fn config_exists(&self, key: Config) -> Result<bool> {
-        Ok(self.sql.get_raw_config(key).await?.is_some())
-    }
-
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
+    pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(key).await?;
+                let rel_path = self.sql.get_raw_config(self, key).await;
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(key).await?,
+            _ => self.sql.get_raw_config(self, key).await,
         };
 
         if value.is_some() {
-            return Ok(value);
+            return value;
         }
 
         // Default values
         match key {
-            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
-            Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
-            _ => Ok(key.get_str("default").map(|s| s.to_string())),
+            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
 
-    pub async fn get_config_int(&self, key: Config) -> Result<i32> {
+    pub async fn get_config_int(&self, key: Config) -> i32 {
         self.get_config(key)
             .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
     }
 
-    pub async fn get_config_i64(&self, key: Config) -> Result<i64> {
-        self.get_config(key)
-            .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
-    }
-
-    pub async fn get_config_u64(&self, key: Config) -> Result<u64> {
-        self.get_config(key)
-            .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
-    }
-
-    pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
-        Ok(self.get_config_int(key).await? != 0)
+    pub async fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key).await != 0
     }
 
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
-    pub async fn get_config_delete_server_after(&self) -> Result<Option<i64>> {
-        match self.get_config_int(Config::DeleteServerAfter).await? {
-            0 => Ok(None),
-            1 => Ok(Some(0)),
-            x => Ok(Some(x as i64)),
+    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter).await {
+            0 => None,
+            1 => Some(0),
+            x => Some(x as i64),
         }
     }
 
-    /// Gets the configured provider, as saved in the `configured_provider` value.
-    ///
-    /// The provider is determined by `get_provider_info()` during configuration and then saved
-    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
-    pub async fn get_configured_provider(&self) -> Result<Option<&'static Provider>> {
-        if let Some(cfg) = self.get_config(Config::ConfiguredProvider).await? {
-            return Ok(get_provider_by_id(&cfg));
-        }
-        Ok(None)
-    }
-
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
     /// after `x` seconds.
-    pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
-        match self.get_config_int(Config::DeleteDeviceAfter).await? {
-            0 => Ok(None),
-            x => Ok(Some(x as i64)),
+    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter).await {
+            0 => None,
+            x => Some(x as i64),
         }
     }
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
                     .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![])
                     .await?;
                 self.sql
-                    .set_raw_config_bool("attach_selfavatar", true)
+                    .set_raw_config_bool(self, "attach_selfavatar", true)
                     .await?;
                 match value {
                     Some(value) => {
-                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
-                        blob.recode_to_avatar_size(self).await?;
-                        self.sql.set_raw_config(key, Some(blob.as_name())).await?;
-                        Ok(())
-                    }
-                    None => {
-                        self.sql.set_raw_config(key, None).await?;
-                        Ok(())
+                        let blob = BlobObject::new_from_path(&self, value).await?;
+                        blob.recode_to_avatar_size(self)?;
+                        self.sql
+                            .set_raw_config(self, key, Some(blob.as_name()))
+                            .await
                     }
+                    None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
+            Config::InboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                self.interrupt_inbox(false).await;
+                ret
+            }
+            Config::SentboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                self.interrupt_sentbox(false).await;
+                ret
+            }
+            Config::MvboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                self.interrupt_mvbox(false).await;
+                ret
+            }
             Config::Selfstatus => {
-                let def = stock_str::status_line(self).await;
+                let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(key, val).await?;
-                Ok(())
+                self.sql.set_raw_config(self, key, val).await
             }
             Config::DeleteDeviceAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
+                let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.emit_event(EventType::MsgsChanged {
+                self.emit_event(Event::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
                 ret
             }
-            Config::Displayname => {
-                let value = value.map(improve_single_line_input);
-                self.sql.set_raw_config(key, value.as_deref()).await?;
-                Ok(())
-            }
-            Config::DeleteServerAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
-                job::schedule_resync(self).await;
-                ret
-            }
-            _ => {
-                self.sql.set_raw_config(key, value).await?;
-                Ok(())
-            }
+            _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
-
-    pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { None })
-            .await?;
-        Ok(())
-    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -331,8 +255,12 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::test_utils::TestContext;
+    use crate::constants::AVATAR_SIZE;
+    use crate::test_utils::*;
+    use image::GenericImageView;
     use num_traits::FromPrimitive;
+    use std::fs::File;
+    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -346,17 +274,104 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_default_prop() {
+        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = dummy_context().await;
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(avatar_blob).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = dummy_context().await;
+        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = dummy_context().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
+
     #[async_std::test]
     async fn test_media_quality_config_option() {
-        let t = TestContext::new().await;
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+        let t = dummy_context().await;
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
+        t.ctx
+            .set_config(Config::MediaQuality, Some("1"))
+            .await
+            .unwrap();
 
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure.rs

@@ -1,641 +0,0 @@
-//! Email accounts autoconfiguration process module
-
-mod auto_mozilla;
-mod auto_outlook;
-mod read_url;
-mod server_params;
-
-use anyhow::{bail, ensure, Context as _, Result};
-use async_std::prelude::*;
-use async_std::task;
-use itertools::Itertools;
-use job::Action;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
-
-use crate::dc_tools::EmailAddress;
-use crate::imap::Imap;
-use crate::login_param::{LoginParam, ServerLoginParam};
-use crate::message::Message;
-use crate::oauth2::dc_get_oauth2_addr;
-use crate::provider::{Protocol, Socket, UsernamePattern};
-use crate::smtp::Smtp;
-use crate::stock_str;
-use crate::{chat, e2ee, provider};
-use crate::{config::Config, dc_tools::time};
-use crate::{
-    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
-    job,
-};
-use crate::{context::Context, param::Params};
-
-use auto_mozilla::moz_autoconfigure;
-use auto_outlook::outlk_autodiscover;
-use server_params::{expand_param_vector, ServerParams};
-
-macro_rules! progress {
-    ($context:tt, $progress:expr, $comment:expr) => {
-        assert!(
-            $progress <= 1000,
-            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
-        );
-        $context.emit_event($crate::events::EventType::ConfigureProgress {
-            progress: $progress,
-            comment: $comment,
-        });
-    };
-    ($context:tt, $progress:expr) => {
-        progress!($context, $progress, None);
-    };
-}
-
-impl Context {
-    /// Checks if the context is already configured.
-    pub async fn is_configured(&self) -> Result<bool> {
-        self.sql
-            .get_raw_config_bool("configured")
-            .await
-            .map_err(Into::into)
-    }
-
-    /// Configures this account with the currently set parameters.
-    pub async fn configure(&self) -> Result<()> {
-        use futures::future::FutureExt;
-
-        ensure!(
-            !self.scheduler.read().await.is_running(),
-            "cannot configure, already running"
-        );
-        ensure!(
-            self.sql.is_open().await,
-            "cannot configure, database not opened."
-        );
-        let cancel_channel = self.alloc_ongoing().await?;
-
-        let res = self
-            .inner_configure()
-            .race(cancel_channel.recv().map(|_| {
-                progress!(self, 0);
-                Ok(())
-            }))
-            .await;
-
-        self.free_ongoing().await;
-
-        res
-    }
-
-    async fn inner_configure(&self) -> Result<()> {
-        info!(self, "Configure ...");
-
-        let mut param = LoginParam::from_database(self, "").await?;
-        let success = configure(self, &mut param).await;
-        self.set_config(Config::NotifyAboutWrongPw, None).await?;
-
-        if let Some(provider) = param.provider {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    if !self.config_exists(def.key).await? {
-                        info!(self, "apply config_defaults {}={}", def.key, def.value);
-                        self.set_config(def.key, Some(def.value)).await?;
-                    } else {
-                        info!(
-                            self,
-                            "skip already set config_defaults {}={}", def.key, def.value
-                        );
-                    }
-                }
-            }
-
-            if !provider.after_login_hint.is_empty() {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(provider.after_login_hint.to_string());
-                if chat::add_device_msg(self, Some("core-provider-info"), Some(&mut msg))
-                    .await
-                    .is_err()
-                {
-                    warn!(self, "cannot add after_login_hint as core-provider-info");
-                }
-            }
-        }
-
-        match success {
-            Ok(_) => {
-                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
-                    .await?;
-                progress!(self, 1000);
-                Ok(())
-            }
-            Err(err) => {
-                progress!(
-                    self,
-                    0,
-                    Some(
-                        stock_str::configuration_failed(
-                            self,
-                            // We are using Anyhow's .context() and to show the
-                            // inner error, too, we need the {:#}:
-                            format!("{:#}", err),
-                        )
-                        .await
-                    )
-                );
-                Err(err)
-            }
-        }
-    }
-}
-
-async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    progress!(ctx, 1);
-
-    // Check basic settings.
-    ensure!(!param.addr.is_empty(), "Please enter an email address.");
-
-    // Only check for IMAP password, SMTP password is an "advanced" setting.
-    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
-    if param.smtp.password.is_empty() {
-        param.smtp.password = param.imap.password.clone()
-    }
-
-    // Normalize authentication flags.
-    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
-        DC_LP_AUTH_OAUTH2 => true,
-        DC_LP_AUTH_NORMAL => false,
-        _ => false,
-    };
-    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-    param.server_flags |= if oauth2 {
-        DC_LP_AUTH_OAUTH2 as i32
-    } else {
-        DC_LP_AUTH_NORMAL as i32
-    };
-
-    let ctx2 = ctx.clone();
-    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
-
-    // Step 1: Load the parameters and check email-address and password
-
-    if oauth2 {
-        // the used oauth2 addr may differ, check this.
-        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
-        progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
-            .await?
-            .and_then(|e| e.parse().ok())
-        {
-            info!(ctx, "Authorized address is {}", oauth2_addr);
-            param.addr = oauth2_addr;
-            ctx.sql
-                .set_raw_config("addr", Some(param.addr.as_str()))
-                .await?;
-        }
-        progress!(ctx, 20);
-    }
-    // no oauth? - just continue it's no error
-
-    let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-    let param_domain = parsed.domain;
-    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
-
-    // Step 2: Autoconfig
-    progress!(ctx, 200);
-
-    let param_autoconfig;
-    if param.imap.server.is_empty()
-        && param.imap.port == 0
-        && param.imap.security == Socket::Automatic
-        && param.imap.user.is_empty()
-        && param.smtp.server.is_empty()
-        && param.smtp.port == 0
-        && param.smtp.security == Socket::Automatic
-        && param.smtp.user.is_empty()
-    {
-        // no advanced parameters entered by the user: query provider-database or do Autoconfig
-
-        info!(
-            ctx,
-            "checking internal provider-info for offline autoconfig"
-        );
-
-        if let Some(provider) = provider::get_provider_info(&param_domain).await {
-            param.provider = Some(provider);
-            match provider.status {
-                provider::Status::Ok | provider::Status::Preparation => {
-                    if provider.server.is_empty() {
-                        info!(ctx, "offline autoconfig found, but no servers defined");
-                        param_autoconfig = None;
-                    } else {
-                        info!(ctx, "offline autoconfig found");
-                        let servers = provider
-                            .server
-                            .iter()
-                            .map(|s| ServerParams {
-                                protocol: s.protocol,
-                                socket: s.socket,
-                                hostname: s.hostname.to_string(),
-                                port: s.port,
-                                username: match s.username_pattern {
-                                    UsernamePattern::Email => param.addr.to_string(),
-                                    UsernamePattern::Emaillocalpart => {
-                                        if let Some(at) = param.addr.find('@') {
-                                            param.addr.split_at(at).0.to_string()
-                                        } else {
-                                            param.addr.to_string()
-                                        }
-                                    }
-                                },
-                            })
-                            .collect();
-
-                        param_autoconfig = Some(servers)
-                    }
-                }
-                provider::Status::Broken => {
-                    info!(ctx, "offline autoconfig found, provider is broken");
-                    param_autoconfig = None;
-                }
-            }
-        } else {
-            info!(ctx, "no offline autoconfig found");
-            param_autoconfig =
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
-        }
-    } else {
-        param_autoconfig = None;
-    }
-
-    progress!(ctx, 500);
-
-    let mut servers = param_autoconfig.unwrap_or_default();
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::Imap)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::Imap,
-            hostname: param.imap.server.clone(),
-            port: param.imap.port,
-            socket: param.imap.security,
-            username: param.imap.user.clone(),
-        })
-    }
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::Smtp)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::Smtp,
-            hostname: param.smtp.server.clone(),
-            port: param.smtp.port,
-            socket: param.smtp.security,
-            username: param.smtp.user.clone(),
-        })
-    }
-    let servers = expand_param_vector(servers, &param.addr, &param_domain);
-
-    progress!(ctx, 550);
-
-    // Spawn SMTP configuration task
-    let mut smtp = Smtp::new();
-
-    let context_smtp = ctx.clone();
-    let mut smtp_param = param.smtp.clone();
-    let smtp_addr = param.addr.clone();
-    let smtp_servers: Vec<ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Smtp)
-        .cloned()
-        .collect();
-    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
-
-    let smtp_config_task = task::spawn(async move {
-        let mut smtp_configured = false;
-        let mut errors = Vec::new();
-        for smtp_server in smtp_servers {
-            smtp_param.user = smtp_server.username.clone();
-            smtp_param.server = smtp_server.hostname.clone();
-            smtp_param.port = smtp_server.port;
-            smtp_param.security = smtp_server.socket;
-
-            match try_smtp_one_param(
-                &context_smtp,
-                &smtp_param,
-                &smtp_addr,
-                oauth2,
-                provider_strict_tls,
-                &mut smtp,
-            )
-            .await
-            {
-                Ok(_) => {
-                    smtp_configured = true;
-                    break;
-                }
-                Err(e) => errors.push(e),
-            }
-        }
-
-        if smtp_configured {
-            Ok(smtp_param)
-        } else {
-            Err(errors)
-        }
-    });
-
-    progress!(ctx, 600);
-
-    // Configure IMAP
-    let (_s, r) = async_std::channel::bounded(1);
-    let mut imap = Imap::new(r);
-
-    let mut imap_configured = false;
-    let imap_servers: Vec<&ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Imap)
-        .collect();
-    let imap_servers_count = imap_servers.len();
-    let mut errors = Vec::new();
-    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
-        param.imap.user = imap_server.username.clone();
-        param.imap.server = imap_server.hostname.clone();
-        param.imap.port = imap_server.port;
-        param.imap.security = imap_server.socket;
-
-        match try_imap_one_param(
-            ctx,
-            &param.imap,
-            &param.addr,
-            oauth2,
-            provider_strict_tls,
-            &mut imap,
-        )
-        .await
-        {
-            Ok(_) => {
-                imap_configured = true;
-                break;
-            }
-            Err(e) => errors.push(e),
-        }
-        progress!(
-            ctx,
-            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
-        );
-    }
-    if !imap_configured {
-        bail!(nicer_configuration_error(ctx, errors).await);
-    }
-
-    progress!(ctx, 850);
-
-    // Wait for SMTP configuration
-    match smtp_config_task.await {
-        Ok(smtp_param) => {
-            param.smtp = smtp_param;
-        }
-        Err(errors) => {
-            bail!(nicer_configuration_error(ctx, errors).await);
-        }
-    }
-
-    progress!(ctx, 900);
-
-    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
-        || ctx.get_config_bool(Config::MvboxMove).await?;
-
-    imap.configure_folders(ctx, create_mvbox).await?;
-
-    imap.select_with_uidvalidity(ctx, "INBOX")
-        .await
-        .context("could not read INBOX status")?;
-
-    drop(imap);
-
-    progress!(ctx, 910);
-    // configuration success - write back the configured parameters with the
-    // "configured_" prefix; also write the "configured"-flag */
-    // the trailing underscore is correct
-    param.save_to_database(ctx, "configured_").await?;
-    ctx.sql.set_raw_config_bool("configured", true).await?;
-    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
-        .await?;
-
-    progress!(ctx, 920);
-
-    e2ee::ensure_secret_key_exists(ctx).await?;
-    info!(ctx, "key generation completed");
-
-    job::add(
-        ctx,
-        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
-    )
-    .await;
-
-    progress!(ctx, 940);
-    update_device_chats_handle.await?;
-
-    Ok(())
-}
-
-/// Retrieve available autoconfigurations.
-///
-/// A Search configurations from the domain used in the email-address, prefer encrypted
-/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
-async fn get_autoconfig(
-    ctx: &Context,
-    param: &LoginParam,
-    param_domain: &str,
-    param_addr_urlencoded: &str,
-) -> Option<Vec<ServerParams>> {
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        &format!(
-            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-            param_domain, param_addr_urlencoded
-        ),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 300);
-
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
-        &format!(
-            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-            &param_domain, &param_addr_urlencoded
-        ),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 310);
-
-    // Outlook uses always SSL but different domains (this comment describes the next two steps)
-    if let Ok(res) = outlk_autodiscover(
-        ctx,
-        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 320);
-
-    if let Ok(res) = outlk_autodiscover(
-        ctx,
-        format!(
-            "https://autodiscover.{}/autodiscover/autodiscover.xml",
-            &param_domain
-        ),
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 330);
-
-    // always SSL for Thunderbird's database
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-
-    None
-}
-
-async fn try_imap_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    addr: &str,
-    oauth2: bool,
-    provider_strict_tls: bool,
-    imap: &mut Imap,
-) -> Result<(), ConfigurationError> {
-    let inf = format!(
-        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
-    );
-    info!(context, "Trying: {}", inf);
-
-    if let Err(err) = imap
-        .connect(context, param, addr, oauth2, provider_strict_tls)
-        .await
-    {
-        info!(context, "failure: {}", err);
-        Err(ConfigurationError {
-            config: inf,
-            msg: err.to_string(),
-        })
-    } else {
-        info!(context, "success: {}", inf);
-        Ok(())
-    }
-}
-
-async fn try_smtp_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    addr: &str,
-    oauth2: bool,
-    provider_strict_tls: bool,
-    smtp: &mut Smtp,
-) -> Result<(), ConfigurationError> {
-    let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
-    );
-    info!(context, "Trying: {}", inf);
-
-    if let Err(err) = smtp
-        .connect(context, param, addr, oauth2, provider_strict_tls)
-        .await
-    {
-        info!(context, "failure: {}", err);
-        Err(ConfigurationError {
-            config: inf,
-            msg: err.to_string(),
-        })
-    } else {
-        info!(context, "success: {}", inf);
-        smtp.disconnect().await;
-        Ok(())
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-#[error("Trying {config}…\nError: {msg}")]
-pub struct ConfigurationError {
-    config: String,
-    msg: String,
-}
-
-async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
-    let first_err = if let Some(f) = errors.first() {
-        f
-    } else {
-        // This means configuration failed but no errors have been captured. This should never
-        // happen, but if it does, the user will see classic "Error: no error".
-        return "no error".to_string();
-    };
-
-    if errors
-        .iter()
-        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
-    {
-        return stock_str::error_no_network(context).await;
-    }
-
-    if errors.iter().all(|e| e.msg == first_err.msg) {
-        return first_err.msg.to_string();
-    }
-
-    errors.iter().map(|e| e.to_string()).join("\n\n")
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("Invalid email address: {0:?}")]
-    InvalidEmailAddress(String),
-
-    #[error("XML error at position {position}: {error}")]
-    InvalidXml {
-        position: usize,
-        #[source]
-        error: quick_xml::Error,
-    },
-
-    #[error("Failed to get URL: {0}")]
-    ReadUrlError(#[from] self::read_url::Error),
-
-    #[error("Number of redirection is exceeded")]
-    RedirectionError,
-}
-
-#[cfg(test)]
-mod tests {
-    #![allow(clippy::indexing_slicing)]
-
-    use crate::config::Config;
-    use crate::test_utils::TestContext;
-
-    #[async_std::test]
-    async fn test_no_panic_on_bad_credentials() {
-        let t = TestContext::new().await;
-        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
-            .await
-            .unwrap();
-        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
-        assert!(t.configure().await.is_err());
-    }
-}