Rustfmt, Comment, move a little bit slower

.circleci/config.yml

@@ -0,0 +1,239 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+  python:
+    docker:
+      - image: 3.7.7-stretch
+            
+
+restore-workspace: &restore-workspace
+  attach_workspace:
+    at: /mnt
+
+restore-cache: &restore-cache
+  restore_cache:
+    keys:
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - repo-source-{{ .Branch }}-{{ .Revision }}
+
+commands:
+  test_target:
+    parameters:
+      target:
+        type: string
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Test (<< parameters.target >>)
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
+          no_output_timeout: 15m
+
+jobs:
+  cargo_fetch:
+    executor: default
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
+      - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
+      - run: cargo fetch
+      - run: rustc +stable --version
+      - run: rustc +$(cat rust-toolchain) --version
+      # make sure this git repo doesn't grow too big
+      - run: git gc
+      - persist_to_workspace:
+          root: /mnt
+          paths:
+            - crate
+      - save_cache:
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          paths:
+            - "~/.cargo"
+            - "~/.rustup"
+
+  rustfmt:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo fmt
+          command: cargo fmt --all -- --check
+
+  test_macos:
+    macos:
+      xcode: "10.0.0"
+    working_directory: ~/crate
+    steps:
+      - run:
+          name: Configure environment variables
+          command: |
+            echo 'export PATH="${HOME}/.cargo/bin:${HOME}/.bin:${PATH}"' >> $BASH_ENV
+            echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV
+      - checkout
+      - run:
+          name: Install Rust
+          command: |
+            curl https://sh.rustup.rs -sSf | sh -s -- -y
+      - run: rustup install $(cat rust-toolchain)
+      - run: rustup default $(cat rust-toolchain)
+      - run: cargo fetch
+      - run:
+          name: Test
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
+
+  test_x86_64-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "x86_64-unknown-linux-gnu"
+
+  test_i686-unknown-linux-gnu:
+    executor: default
+    steps:
+      - test_target:
+          target: "i686-unknown-linux-gnu"
+
+  test_aarch64-linux-android:
+    executor: default
+    steps:
+      - test_target:
+          target: "aarch64-linux-android"
+
+
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: pyenv versions
+      - run: pyenv global 3.5.2
+      - run: ls -laR workspace
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+  clippy:
+    executor: default
+    steps:
+      - *restore-workspace
+      - *restore-cache
+      - run:
+          name: Run cargo clippy
+          command: cargo clippy 
+
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      # - cargo_fetch
+
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
+
+      - build_doxygen:
+          filters:
+            branches:
+              only: master
+            tags:
+              only: /.*/
+
+      # Linux Desktop 64bit
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Linux Desktop 32bit
+      # - test_i686-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
+
+      # Android 64bit
+      # - test_aarch64-linux-android:
+      #     requires:
+      #       - cargo_fetch
+
+      # Desktop Apple
+      # - test_macos:
+      #     requires:
+      #       - cargo_fetch

.gitattributes

@@ -12,6 +12,3 @@ test-data/* text=false
 *.gif  binary
 *.ico  binary
 
-*.py diff=python
-*.rs diff=rust
-*.md diff=markdown

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "daily"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 10

.github/workflows/ci.yml

@@ -14,11 +14,11 @@ jobs:
     name: Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.45.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -29,10 +29,10 @@ jobs:
   run_clippy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.45.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -40,52 +40,29 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --workspace --tests --examples
 
-  docs:
-    name: Rust doc comments
-    runs-on: ubuntu-latest
-    env:
-      RUSTDOCFLAGS: -Dwarnings
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.54.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.54.0
-            python: false # Python bindings compilation on Windows is not supported.
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
+        rust: [1.45.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
 
-          # Minimum Supported Rust Version = 1.48.0
-          # This is the Debian "bullseye" release version of Rust.
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.48.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -115,10 +92,8 @@ jobs:
 
     - name: check
       uses: actions-rs/cargo@v1
-      env:
-        RUSTFLAGS: -D warnings
       with:
-        command: check
+        command:  check
         args: --all --bins --examples --tests --features repl
 
     - name: tests
@@ -126,29 +101,3 @@ jobs:
       with:
         command: test
         args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,doc,py3

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.gitignore

@@ -1,6 +1,5 @@
 /target
 **/*.rs.bk
-/build
 
 # ignore vi temporaries
 *~
@@ -26,5 +25,3 @@ deltachat-ffi/html
 deltachat-ffi/xml
 
 .rsynclist
-
-coverage/

CHANGELOG.md

@@ -1,501 +1,5 @@
 # Changelog 
 
-## 1.59.0
-
-### Added
-- add quota information to `dc_get_connectivity_html()`
-
-### Changes
-- refactorings #2592 #2570 #2581
-- add 'device chat about' to now existing status #2613
-- update provider database #2608
-
-### Fixes
-- provider database supports socket=PLAIN and dotless domains now #2604 #2608
-- add migrated accounts to events emitter #2607
-- fix forwarding quote-only mails #2600
-- do not set WantsMdn param for outgoing messages #2603
-- set timestamps for system messages #2593
-- do not treat gmail labels as folders #2587
-- avoid timing problems in `dc_maybe_network_lost()` #2551
-- only set smtp to "connected" if the last message was actually sent #2541
-
-
-## 1.58.0
-
-### Fixes
-
-- move WAL file together with database
-  and avoid using data if the database was not closed correctly before #2583
-
-
-## 1.57.0
-
-### API Changes
-
-- breaking change: removed deaddrop chat #2514 #2563
-
-  Contact request chats are not merged into a single virtual
-  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
-  same way as other chats, but sending of messages to them is not
-  allowed and MDNs are not sent automatically until the chat is
-  "accepted" by the user.
-
-  New API:
-  - `dc_chat_is_contact_request()`: returns true if chat is a contact
-    request.  In this case an option to accept the chat via
-    `dc_accept_chat()` should be shown in the UI.
-  - `dc_accept_chat()`: unblock the chat or accept contact request
-  - `dc_block_chat()`: block the chat, currently works only for mailing
-    lists.
-
-  Removed API:
-  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_decide_on_contact_request()`: this call requires a message ID
-    from deaddrop chat as input. As deaddrop chat is removed, this
-    call can't be used anymore.
-  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
-    only difference between these calls was in handling of deaddrop
-    chat
-  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
-
-- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
-  Instead, there is a new api `dc_get_connectivity()`
-  and `dc_get_connectivity_html()`;
-  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
-
-- breaking change: removed `dc_accounts_import_account()`
-  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
-  on its context
-
-- update account api, 2 new methods:
-  `int dc_all_work_done (dc_context_t* context);`
-  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
-
-- add api to check if a message was `Auto-Submitted`
-  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
-  python: `Message.is_bot()`
-
-- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
-  now returns `NULL` if there is no selected account
-
-- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
-  connectivity loss on its own (eg. iOS) #2550
-
-### Added
-- use Auto-Submitted: auto-generated header to identify bots #2502
-- allow sending stickers via repl tool
-- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
-- withdraw/revive own qr-codes #2512
-- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
-
-### Changes
-- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
-- update documentation #2548 #2561 #2569
-- breaking: `Accounts::create` does not also create an default account anymore #2500
-- remove "forwarded" from stickers, as the primary way of getting stickers
-  is by asking a bot and then forwarding them currently #2526
-- mimeparser: use mailparse to parse RFC 2231 filenames #2543
-- allow email addresses without dot in the domain part #2112
-- allow installing lib and include under different prefixes #2558
-- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
-- improve tests #2487 #2491 #2497
-- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
-- improve ci #2494
-- update provider-database #2565
-
-### Removed
-- remove `dc_accounts_import_account()` api #2521
-- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
-
-### Fixes
-- allow stickers with gif-images #2481
-- fix database migration #2486
-- do not count hidden messages in get_msg_cnt(). #2493
-- improve drafts detection #2489
-- fix panic when removing last, selected account from account manager #2500
-- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
-- avoid hiding outgoing classic emails #2505
-- fixes for message timestamps #2517
-- do not process names, avatars, location XMLs, message signature etc.
-  for duplicate messages #2513
-- fix `can_send` for users not in group #2479
-- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
-- fix which chats messages are assigned to #2465
-- fix: don't create chats when MDNs are received #2578
-
-
-## 1.56.0
-
-- fix downscaling images #2469
-
-- fix outgoing messages popping up in selfchat #2456
-
-- securejoin: display error reason if there is any #2470
-
-- do not allow deleting contacts with ongoing chats #2458
-
-- fix: ignore drafts folder when scanning #2454
-
-- fix: scan folders also when inbox is not watched #2446
-
-- more robust In-Reply-To parsing #2182
-
-- update dependencies #2441 #2438 #2439 #2440 #2447 #2448 #2449 #2452 #2453 #2460 #2464 #2466
-
-- update provider-database #2471
-
-- refactorings #2459 #2457
-
-- improve tests and ci #2445 #2450 #2451
-
-
-## 1.55.0
-
-- fix panic when receiving some HTML messages #2434
-
-- fix downloading some messages multiple times #2430
-
-- fix formatting of read receipt texts #2431
-
-- simplify SQL error handling #2415
-
-- explicit rust API for creating chats with blocked status #2282
-
-- debloat the binary by using less AsRef arguments #2425
-
-
-## 1.54.0
-
-- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
-
-- global search performance improvement #2364 #2365 #2366
-
-- improve SQLite performance with `PRAGMA synchronous=normal` #2382
-
-- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
-
-- python: list `requests` as a requirement #2390
-
-- fix creation of many delete jobs when being offline #2372
-
-- synchronize status between devices #2386
-
-- deaddrop (contact requests) chat improvements #2373
-
-- add "Forwarded:" to notification and chatlist summaries #2310
-
-- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
-
-- improve tests #2360 #2362 #2370 #2377 #2387
-
-- cleanup #2359 #2361 #2374 #2376 #2379 #2388
-
-
-## 1.53.0
-
-- fix sqlx performance regression #2355 2356
-
-- add a `ci_scripts/coverage.sh` #2333 #2334
-
-- refactorings and tests #2348 #2349 #2350
-
-- improve python bindings #2332 #2326
-
-
-## 1.52.0
-
-- database library changed from rusqlite to sqlx #2089 #2331 #2336 #2340
-
-- add alias support: UIs should check for `dc_msg_get_override_sender_name()`
-  also in single-chats now and display divergent names and avatars #2297
-
-- parse blockquote-tags for better quote detection #2313
-
-- ignore unknown classical emails from spam folder #2311
-
-- support "Mixed Up” encryption repairing #2321
-
-- fix single chat search #2344
-
-- fix nightly clippy and rustc errors #2341
-
-- update dependencies #2350
-
-- improve ci #2342
-
-- improve python bindings #2332 #2326
-
-
-## 1.51.0
-
-- breaking change: You have to call `dc_stop_io()`/`dc_start_io()`
-  before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`:
-  fix race condition and db corruption
-  when a message was received during backup #2253
-
-- save subject for messages: new api `dc_msg_get_subject()`,
-  when quoting, use the subject of the quoted message as the new subject,
-  instead of the last subject in the chat #2274 #2283
-
-- new apis to get full or html message,
-  `dc_msg_has_html()` and `dc_get_msg_html()` #2125 #2151 #2264 #2279
-
-- new chat type and apis for the new mailing list support,
-  `DC_CHAT_TYPE_MAILINGLIST`, `dc_msg_get_real_chat_id()`,
-  `dc_msg_get_override_sender_name()` #1964 #2181 #2185 #2195 #2211 #2210 #2240
-  #2241 #2243 #2258 #2259 #2261 #2267 #2270 #2272 #2290
-
-- new api `dc_decide_on_contact_request()`,
-  deprecated `dc_create_chat_by_msg_id()` and `dc_marknoticed_contact()` #1964
-
-- new flag `DC_GCM_INFO_ONLY` for api `dc_get_chat_msgs()` #2132
-
-- new api `dc_get_chat_encrinfo()` #2186
-
-- new api `dc_contact_get_status()`, returning the recent footer #2218 #2307
-
-- improve contact name update rules,
-  add api `dc_contact_get_auth_name()` #2206 #2212 #2225
-
-- new api for bots: `dc_msg_set_html()` #2153
-
-- new api for bots: `dc_msg_set_override_sender_name()` #2231
-
-- api removed: `dc_is_io_running()` #2139
-
-- api removed: `dc_contact_get_first_name()` #2165 #2171
-
-- improve compatibility with providers changing the Message-ID
-  (as Outlook.com) #2250 #2265
-
-- correctly show emails that were sent to an alias and then bounced 
-
-- implement Consistent Color Generation (XEP-0392),
-  that results in contact colors be be changed #2228 #2229 #2239
-
-- fetch recent existing messages
-  and create corresponding chats after configure #2106
-
-- improve e-mail compatibility
-  by scanning all folders from time to time #2067 #2152 #2158 #2184 #2215 #2224
-
-- better support videochat-services not supporting random rooms #2191
-
-- export backups as .tar files #2023
-
-- scale avatars based on media_quality, fix avatar rotation #2063
-
-- compare ephemeral timer to parent message to deal with reordering better #2100
-
-- better ephemeral system messages #2183
-
-- read quotes out of html messages #2104
-
-- prepend subject to messages with attachments, if needed #2111
-
-- run housekeeping at least once a day #2114
-
-- resolve MX domain only once per OAuth2 provider #2122
-
-- configure provider based on MX record #2123 #2134
-
-- make transient bad destination address error permanent
-  after n tries #2126 #2202
-
-- enable strict TLS for known providers by default #2121
-
-- improve and harden secure join #2154 #2161 #2251
-
-- update `dc_get_info()` to return more information #2156
-
-- prefer In-Reply-To/References
-  over group-id stored in Message-ID #2164 #2172 #2173
-
-- apply gossiped encryption preference to new peerstates #2174
-
-- fix: do not return quoted messages from the trash chat #2221
-
-- fix: allow emojis for location markers #2177
-
-- fix encoding of Chat-Group-Name-Changed messages that could even lead to
-  messages not being delivered #2141
-
-- fix error when no temporary directory is available #1929
-
-- fix marking read receipts as seen #2117
-
-- fix read-notification for mixed-case addresses #2103
-
-- fix decoding of attachment filenames #2080 #2094 #2102
-
-- fix downloading ranges of message #2061
-
-- fix parsing quoted encoded words in From: header #2193 #2204
-
-- fix import/export race condition #2250
-
-- fix: exclude muted chats from notified-list #2269 #2275
-
-- fix: update uid_next if the server rewind it #2288
-
-- fix: return error on fingerprint mismatch on qr-scan #2295
-
-- fix ci #2217 #2226 #2244 #2245 #2249 #2277 #2286
-
-- try harder on backup opening #2148
-
-- trash messages more thoroughly #2273
-
-- nicer logging #2284
-
-- add CMakeLists.txt #2260
-
-- switch to rust 1.50, update toolchains, deps #2150 #2155 #2165 #2107 #2262 #2271
-
-- improve python bindings #2113 #2115 #2133 #2214
-
-- improve documentation #2143 #2160 #2175 #2146
-
-- refactorings #2110 #2136 #2135 #2168 #2178 #2189 #2190 #2198 #2197 #2201 #2196
-  #2200 #2230 #2262 #2203
-
-- update provider-database #2299
-
-
-## 1.50.0
-
-- do not fetch emails in between inbox_watch disabled and enabled again #2087
-
-- fix: do not fetch from INBOX if inbox_watch is disabled #2085
-
-- fix: do not use STARTTLS when PLAIN connection is requested
-  and do not allow downgrade if STARTTLS is not available #2071
-
-
-## 1.49.0
-
-- add timestamps to image and video filenames #2068
-
-- forbid quoting messages from another context #2069
-
-- fix: preserve quotes in messages with attachments #2070
-
-
-## 1.48.0
-
-- `fetch_existing` renamed to `fetch_existing_msgs` and disabled by default
-  #2035 #2042
-
-- skip fetch existing messages/contacts if config-option `bot` set #2017
-
-- always log why a message is sorted to trash #2045
-
-- display a quote if top posting is detected #2047
-
-- add ephemeral task cancellation to `dc_stop_io()`;
-  before, there was no way to quickly terminate pending ephemeral tasks #2051
-
-- when saved-messages chat is deleted,
-  a device-message about recreation is added #2050
-
-- use `max_smtp_rcpt_to` from provider-db,
-  sending messages to many recipients in configurable chunks #2056
-
-- fix handling of empty autoconfigure files #2027
-
-- fix adding saved messages to wrong chats on multi-device #2034 #2039
-
-- fix hang on android4.4 and other systems
-  by adding a workaround to executer-blocking-handling bug #2040
-
-- fix secret key export/import roundtrip #2048
-
-- fix mistakenly unarchived chats #2057
-
-- fix outdated-reminder test that fails only 7 days a year,
-  including halloween :) #2059
-
-- improve python bindings #2021 #2036 #2038
-
-- update provider-database #2037
-
-
-## 1.47.0
-
-- breaking change: `dc_update_device_chats()` removed;
-  this is now done automatically during configure
-  unless the new config-option `bot` is set #1957
-
-- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
-  and remove `dc_marknoticed_all_chats()` #1942 #1981
-
-- breaking change: remove unused starring options #1965
-
-- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
-  `dc_chat_is_protected()`; also single-chats may be protected now, this may
-  happen over the wire even if the UI do not offer an option for that #1968
-
-- breaking change: split quotes off message text,
-  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
-
-- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
-  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
-
-- require quorum to enable encryption #1946
-
-- speed up and clean up account creation #1912 #1927 #1960 #1961
-
-- configure now collects recent contacts and fetches last messages
-  unless disabled by `fetch_existing` config-option #1913 #2003
-  EDIT: `fetch_existing` renamed to `fetch_existing_msgs` in 1.48.0 #2042
-
-- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
-  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
-
-- add `dc_set_chat_protection()`; the `protect` parameter in
-  `dc_create_group_chat()` will be removed in an upcoming release;
-  up to then, UIs using the "verified group" paradigm
-  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
-
-- remove unneeded `DC_STR_COUNT` #1991
-
-- mark all failed messages as failed when receiving an NDN #1993
-
-- check some easy cases for bad system clock and outdated app #1901
-
-- fix import temporary directory usage #1929
-
-- fix forcing encryption for reset peers #1998
-
-- fix: do not allow to save drafts in non-writeable chats #1997
-
-- fix: do not show HTML if there is no content and there is an attachment #1988
-
-- fix recovering offline/lost connections, fixes background receive bug #1983
-
-- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
-
-- fix whitespace for summaries #1938
-
-- fix: improve sentbox name guessing #1941
-
-- fix: avoid manual poll impl for accounts events #1944
-
-- fix encoding newlines in param as a preparation for storing quotes #1945
-
-- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
-
-- fix ci #1928 #1931 #1932 #1933 #1934 #1943
-
-- update provider-database #1940 #2005 #2006
-
-- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
-
-
 ## 1.46.0
 
 - breaking change: `dc_configure()` report errors in
@@ -1241,3 +745,4 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

CMakeLists.txt

@@ -1,42 +0,0 @@
-cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-
-find_program(CARGO cargo)
-
-add_custom_command(
-	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
-)
-
-add_custom_target(
-	lib_deltachat
-	ALL
-	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
-	"target/release/pkgconfig/deltachat.pc"
-)
-
-include(GNUInstallDirs)
-install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,90 +1,83 @@
 [package]
 name = "deltachat"
-version = "1.59.0"
+version = "1.46.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
-[profile.dev]
-debug = 0
-
 [profile.release]
 lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
-ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1.0.42"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20.3"
-async-std = { version = "~1.9.0", features = ["unstable"] }
-async-tar = "0.3.0"
-async-trait = "0.1.50"
-backtrace = "0.3.59"
-base64 = "0.13"
-bitflags = "1.3.1"
-byteorder = "1.3.1"
-chrono = "0.4.6"
-dirs = { version = "3.0.2", optional=true }
-email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1.1"
-futures = "0.3.16"
+libc = "0.2.51"
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7.0"
-itertools = "0.10.1"
-kamadak-exif = "0.5"
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2.98"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13.5"
-native-tls = "0.2.3"
-num_cpus = "1.13.0"
+sha2 = "0.9.0"
+rand = "0.7.0"
+smallvec = "1.0.0"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-once_cell = "1.8.0"
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+async-imap = "0.4.0"
+async-native-tls = { version = "0.3.3" }
+async-std = { version = "1.6.4", features = ["unstable"] }
+base64 = "0.12"
+charset = "0.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7.0", default-features = false }
-pretty_env_logger = { version = "0.4.0", optional = true }
-quick-xml = "0.22.0"
-r2d2 = "0.8.9"
-r2d2_sqlite = "0.18.0"
-rand = "0.7.0"
-regex = "1.4.6"
-rusqlite = "0.25"
-rust-hsluv = "0.1.4"
-rustyline = { version = "8.2.0", optional = true }
-sanitize-filename = "0.3.0"
-serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9.7"
-sha2 = "0.9.5"
-smallvec = "1.0.0"
-stop-token = "0.2.0"
-strum = "0.21.0"
-strum_macros = "0.21.1"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
-thiserror = "1.0.26"
-toml = "0.5.6"
-url = "2.2.2"
+serde_json = "1.0"
+chrono = "0.4.6"
+indexmap = "1.3.0"
+kamadak-exif = "0.5"
+lazy_static = "1.4.0"
+regex = "1.1.6"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
+r2d2 = "0.8.5"
+strum = "0.19.0"
+strum_macros = "0.19.0"
+backtrace = "0.3.33"
+byteorder = "1.3.1"
+itertools = "0.9.0"
+quick-xml = "0.18.1"
+escaper = "0.1.0"
+bitflags = "1.1.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.13.0"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+futures = "0.3.4"
+thiserror = "1.0.14"
+anyhow = "1.0.28"
+async-trait = "0.1.31"
+url = "2.1.1"
+async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4.2"
-humansize = "1.1.1"
+
+pretty_env_logger = { version = "0.4.0", optional = true }
+log = {version = "0.4.8", optional = true }
+rustyline = { version = "4.1.0", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
+toml = "0.5.6"
+
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
-criterion = "0.3"
-futures-lite = "1.12.0"
-log = "0.4.11"
-pretty_assertions = "0.7.2"
-pretty_env_logger = "0.4.0"
-proptest = "1.0"
 tempfile = "3.0"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
 
 [workspace]
 members = [
@@ -107,17 +100,10 @@ required-features = ["repl"]
 name = "create_account"
 harness = false
 
-[[bench]]
-name = "contacts"
-harness = false
-
-[[bench]]
-name = "search_msgs"
-harness = false
-
 [features]
-default = ["vendored"]
+default = []
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
+

README.md

@@ -2,7 +2,7 @@
 
 > Deltachat-core written in Rust 
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
 ## Installing Rust and Cargo
 
@@ -17,7 +17,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -79,16 +79,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -105,7 +95,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests
@@ -121,6 +111,11 @@ $ cargo test -- --ignored
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+[circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
+[circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
+[appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
+[appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
@@ -128,7 +123,7 @@ Language bindings are available for:
 - [C](https://c.delta.chat)
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
-- [Go](https://github.com/deltachat/go-deltachat/)
+- [Go](https://github.com/hugot/go-deltachat/)
 - [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

benches/contacts.rs

@@ -1,39 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::contact::Contact;
-use deltachat::context::Context;
-use tempfile::tempdir;
-
-async fn address_book_benchmark(n: u32, read_count: u32) {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
-        .collect::<Vec<String>>()
-        .join("");
-
-    Contact::add_address_book(&context, &book).await.unwrap();
-
-    let query: Option<&str> = None;
-    for _ in 0..read_count {
-        Contact::get_all(&context, 0, query).await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
-    });
-
-    c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
-    });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -1,29 +0,0 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
-use std::path::Path;
-
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
-    let id = 100;
-    let context = Context::new("FakeOS".into(), dbfile.into(), id)
-        .await
-        .unwrap();
-
-    for _ in 0..10u32 {
-        context.search_msgs(None, "hello").await.unwrap();
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
-        });
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

ci_scripts/README.md

@@ -1,12 +1,11 @@
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Continuous Integration, run through CircleCI and an own build machine.
 
 ## Description of scripts 
 
-- `../.github/workflows` contains jobs run by GitHub Actions.
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
 
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
@@ -27,8 +26,8 @@ There is experimental support for triggering a remote Python or Rust test run
 from your local checkout/branch. You will need to be authorized to login to 
 the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-    scripts/manual_remote_tests.sh rust
-    scripts/manual_remote_tests.sh python
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
@@ -42,10 +41,6 @@ python tests and build wheels (binary packages for Python)
 You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
-    cd scripts  # where all CI things are 
+    cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

ci_scripts/ci_upload.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+
+export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+pushd $WHEELHOUSEDIR
+
+pip3 install -U pip setuptools
+pip3 install devpi-client
+devpi use https://m.devpi.net
+devpi login dc --password $DEVPI_LOGIN
+
+N_BRANCH=${BRANCH//[\/]}
+
+devpi use dc/$N_BRANCH || {
+    devpi index -c $N_BRANCH 
+    devpi use dc/$N_BRANCH
+}
+devpi index $N_BRANCH bases=/root/pypi
+devpi upload deltachat*
+
+popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

ci_scripts/docker-coredeps/deps/build_python.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x -e
+
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
+
+pushd /usr/bin
+
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
+
+popd

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e -x
+
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
+export PATH=/root/.cargo/bin:$PATH
+rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,50 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    shopt -s huponexit
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -1,9 +1,10 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,7 +18,7 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
@@ -28,6 +29,7 @@ ssh $SSHTARGET <<_HERE
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
@@ -41,5 +43,5 @@ ssh $SSHTARGET <<_HERE
     source \$HOME/venv/bin/activate
     which python
 
-    bash scripts/run-python-test.sh 
+    bash ci_scripts/run-python-test.sh 
 _HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,30 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
+    cd $BUILDDIR
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run_all.sh

@@ -19,17 +19,14 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp36-cp36m/bin
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.6
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
 ln -s /opt/python/cp36-cp36m/bin/python3.6
-rm -f python3.7
 ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -42,8 +39,9 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
+unset DCC_PY_LIVECONFIG
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.59.0"
+version = "1.46.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,9 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.9.0"
-anyhow = "1.0.42"
-thiserror = "1.0.26"
+async-std = "1.6.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 rand = "0.7.3"
 
 [features]

deltachat-ffi/DoxygenLayout.xml

@@ -1,38 +0,0 @@
-<doxygenlayout version="1.0">
-  <!-- Generated by doxygen 1.8.20 -->
-  <!-- Navigation index tabs for HTML output -->
-  <navindex>
-    <tab type="mainpage" visible="yes" title=""/>
-    <tab type="classes" visible="yes" title="">
-      <tab type="classlist" visible="no" title="" intro=""/>
-      <tab type="classindex" visible="no" title=""/>
-      <tab type="hierarchy" visible="no" title="" intro=""/>
-      <tab type="classmembers" visible="no" title="" intro=""/>
-    </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
-    <tab type="pages" visible="yes" title="" intro=""/>
-    <tab type="namespaces" visible="yes" title="">
-      <tab type="namespacelist" visible="yes" title="" intro=""/>
-      <tab type="namespacemembers" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="interfaces" visible="yes" title="">
-      <tab type="interfacelist" visible="yes" title="" intro=""/>
-      <tab type="interfaceindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="interfacehierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="structs" visible="yes" title="">
-      <tab type="structlist" visible="yes" title="" intro=""/>
-      <tab type="structindex" visible="$ALPHABETICAL_INDEX" title=""/>
-    </tab>
-    <tab type="exceptions" visible="yes" title="">
-      <tab type="exceptionlist" visible="yes" title="" intro=""/>
-      <tab type="exceptionindex" visible="$ALPHABETICAL_INDEX" title=""/>
-      <tab type="exceptionhierarchy" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="files" visible="yes" title="">
-      <tab type="filelist" visible="yes" title="" intro=""/>
-      <tab type="globals" visible="yes" title="" intro=""/>
-    </tab>
-    <tab type="examples" visible="yes" title="" intro=""/>
-  </navindex>
-</doxygenlayout>

deltachat-ffi/build.rs

@@ -23,13 +23,11 @@ fn main() {
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
         prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -167,20 +170,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +186,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name
@@ -250,7 +241,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 /// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
 /// requires that the pointer contains valid UTF-8 on Windows.
 ///
-/// Because this returns a reference the [Path] slice can not outlive
+/// Because this returns a reference the [Path] silce can not outlive
 /// the original pointer.
 ///
 /// [Path]: std::path::Path

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.74"
+syn = "1.0.13"
 quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -35,11 +35,7 @@ pub fn from_sql_derive(input: TokenStream) -> TokenStream {
         impl rusqlite::types::FromSql for #name {
             fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
                 let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
+                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
             }
         }
     };

draft/group-sync.rst

@@ -5,95 +5,69 @@ Problem: missing eventual group consistency
 If group members are concurrently adding new members,
 the new members will miss each other's additions, example:
 
-1. Alice and Bob are in a two-member group
+- Alice and Bob are in a two-member group
 
-2. Then Alice adds Carol, while concurrently Bob adds Doris
+- Alice adds Carol, concurrently Bob adds Doris
 
-Right now, the group has inconsistent memberships:
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
 
-- Alice and Carol see a (Alice, Carol, Bob) group
-
-- Bob and Doris see a (Bob, Doris, Alice)
-
-This then leads to "sender is unknown" messages in the chat,
-for example when Alice receives a message from Doris,
-or when Bob receives a message from Carol.
-
-There are also other sources for group membership inconsistency:
-
-- leaving/deleting/adding in larger groups, while being offline,
-  increases chances for inconsistent group membership
-
-- dropped group-membership messages
-
-- group-membership messages landing in "Spam"
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
 
 
-Note that all these problems (can) also happen with verified groups,
-then raising "false alarms" which could lure people to ignore such issues.
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
 
-IOW, it's clear we need to do something about it to improve overall
-reliability in group-settings.
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
 
 
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
 
-Solution: replay group modification messages on inconsistencies
-------------------------------------------------------------------
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
 
-For brevity let's abbreviate "group membership modification" as **GMM**.
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
 
-Delta chat has explicit GMM messages, typically encrypted to the group members
-as seen by the device that sends the GMM. The `Spec <https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members>`_ details the Mime headers and format.
+Notes:
 
-If we detect membership inconsistencies we can resend relevant GMM messages
-to the respective chat.  The receiving devices can process those GMM messages
-as if it would be an incoming message. If for example they have already seen
-the Message-ID of the GMM message, they will ignore the message. It's
-probably useful to record GMM message in their original MIME-format and
-not invent a new recording format. Few notes on three aspects:
+- mechanism works for both encrypted and unencrypted add/del messages
 
-- **group-membership-tracking**: All valid GMM messages are persisted in
-  their full raw (signed/encrypted?) MIME-format in the DB. Note that GMM messages
-  already are in the msgs table, and there is a mime_header column which we could
-  extend to contain the raw Mime GMM message.
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
 
-- **consistency_checking**: If an incoming GMM has a member list which is
-  not consistent with our own view, broadcast a "Group-Member-Correction"
-  message to all members containing a multipart list of GMMs.
+- multiple member-added/removed messages can be attached in a single
+  correction message
 
-- **correcting_memberships**: Upon receiving a Group-Member-Correction
-  message we pass the contained GMMs to the "incoming mail pipeline"
-  (without **consistency_checking** them, to avoid recursion issues)
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
 
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
 
-Alice/Carol and Bob/Doris getting on the same page
-++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Recall that Alice/Carol and Bob/Doris had a differening view of
-group membership. With the proposed solution, when Bob receives
-Alice's "Carol added" message, he will notice that Alice (and thus
-also carol) did not know about Doris.  Bob's device sends a
-"Chat-Group-Member-Correction" message containing his own GMM
-when adding Doris. Therefore, the group's membership is healed
-for everyone in a single broadcast message.
-
-Alice might also send a Group-member-Correction message,
-so there is a second chance that the group gets to know all GMMs.
-
-Note, for example, that if for some reason Bobs and Carols provider
-drop GMM messages between them (spam) that Alice and Doris can heal
-it by resending GMM messages whenever they detect them to be out of sync.
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
 
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
 
 Discussions of variants
 ++++++++++++++++++++++++
 
-- instead of acting on GMM messages we could send corrections
-  for any received message that addresses inconsistent group members but
-  a) this could delay group-membership healing
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
   b) could lead to a lot of members sending corrections
-  c) means we might rely on "To-Addresses" which we also like to strike
-     at least for protected chats.
 
 - instead of broadcasting correction messages we could only send it to
   the sender of the inconsistent member-added/removed message.
@@ -109,3 +83,44 @@ Discussions of variants
   while both being in an offline or bad-connection situation).
 
 
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

examples/repl/cmdline.rs

@@ -2,20 +2,18 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Error};
+use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{
-    self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
-};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
+use deltachat::error::Error;
 use deltachat::imex::*;
 use deltachat::location;
-use deltachat::log::LogExt;
 use deltachat::lot::LotState;
 use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
@@ -23,8 +21,6 @@ use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::EventType;
 use deltachat::{config, provider};
-use std::fs;
-use std::time::{Duration, SystemTime};
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
@@ -120,11 +116,11 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         real_spec = spec.to_string();
         context
             .sql()
-            .set_raw_config("import_spec", Some(&real_spec))
+            .set_raw_config(context, "import_spec", Some(&real_spec))
             .await
             .unwrap();
     } else {
-        let rs = context.sql().get_raw_config("import_spec").await.unwrap();
+        let rs = context.sql().get_raw_config(context, "import_spec").await;
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
             return false;
@@ -175,11 +171,8 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let contact = Contact::get_by_id(context, msg.get_from_id())
         .await
         .expect("invalid contact");
-    let contact_name = if let Some(name) = msg.get_override_sender_name() {
-        format!("~{}", name)
-    } else {
-        contact.get_display_name().to_string()
-    };
+
+    let contact_name = contact.get_name();
     let contact_id = contact.get_id();
 
     let statestr = match msg.get_state() {
@@ -192,7 +185,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -200,8 +193,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == 1 {
+        if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -259,11 +251,15 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error>
 }
 
 async fn log_contactlist(context: &Context, contacts: &[u32]) {
+    let mut contacts = contacts.to_vec();
+    if !contacts.contains(&1) {
+        contacts.push(1);
+    }
     for contact_id in contacts {
         let line;
         let mut line2 = "".to_string();
-        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
-            let name = contact.get_display_name();
+        if let Ok(contact) = Contact::get_by_id(context, contact_id).await {
+            let name = contact.get_name();
             let addr = contact.get_addr();
             let verified_state = contact.is_verified(context).await;
             let verified_str = if VerifiedStatus::Unverified != verified_state {
@@ -292,14 +288,14 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
             let peerstate = Peerstate::from_addr(context, &addr)
                 .await
                 .expect("peerstate error");
-            if peerstate.is_some() && *contact_id != 1 {
+            if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
                     peerstate.as_ref().unwrap().prefer_encrypt
                 );
             }
 
-            println!("Contact#{}: {}{}", *contact_id, line, line2);
+            println!("Contact#{}: {}{}", contact_id, line, line2);
         }
     }
 }
@@ -351,7 +347,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -360,8 +355,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listarchived\n\
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
+                 createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createprotected <name>\n\
+                 createverified <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -372,10 +368,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  dellocations\n\
                  getlocations [<contact-id>]\n\
                  send <text>\n\
+                 send-garbage\n\
                  sendimage <file> [<text>]\n\
-                 sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
-                 sendhtml <file for html-part> [<text for plain-part>]\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -384,17 +379,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
-                 mute <chat-id> [<seconds>]\n\
-                 unmute <chat-id>\n\
-                 protect <chat-id>\n\
-                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
-                 accept <chat-id>\n\
-                 decline <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -406,14 +394,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  contactinfo <contact-id>\n\
                  delcontact <contact-id>\n\
                  cleanupcontacts\n\
-                 block <contact-id>\n\
-                 unblock <contact-id>\n\
-                 listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
-                 setqr <qr-content>\n\
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
@@ -457,20 +441,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportBackup, dir.as_ref()).await?;
+            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(&context, ImexMode::ImportBackup, arg1.as_ref()).await?;
+            imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref()).await?;
+            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref()).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -510,25 +494,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "info" => {
             println!("{:#?}", context.get_info().await);
         }
-        "connectivity" => {
-            let file = dirs::home_dir()
-                .unwrap_or_default()
-                .join("connectivity.html");
-            match context.get_connectivity_html().await {
-                Ok(html) => {
-                    fs::write(&file, html)?;
-                    println!("Report written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get connectivity html: {}", err);
-                }
-            }
-        }
         "maybenetwork" => {
             context.maybe_network().await;
         }
         "housekeeping" => {
-            sql::housekeeping(&context).await.ok_or_log(&context);
+            sql::housekeeping(&context).await;
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
@@ -540,7 +510,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -551,25 +523,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}{}{}",
+                        "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat.get_id().get_fresh_msg_cnt(&context).await?,
-                        if chat.is_muted() { "🔇" } else { "" },
+                        chat.get_id().get_fresh_msg_cnt(&context).await,
                         match chat.visibility {
                             ChatVisibility::Normal => "",
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
-                        if chat.is_protected() { "🛡️" } else { "" },
-                        if chat.is_contact_request() {
-                            "🆕"
-                        } else {
-                            ""
-                        },
                     );
-                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await?;
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
@@ -602,7 +567,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await {
+            if location::is_sending_locations_to_chat(&context, ChatId::new(0)).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -622,11 +587,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let time_start = std::time::SystemTime::now();
-            let msglist =
-                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
-
+            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
@@ -636,41 +597,33 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 })
                 .collect();
 
-            let members = chat::get_chat_contacts(&context, sel_chat.id).await?;
+            let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
             } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
                 let contact = Contact::get_by_id(&context, members[0]).await?;
                 contact.get_addr().to_string()
-            } else if sel_chat.get_type() == Chattype::Mailinglist && !members.is_empty() {
-                "mailinglist".to_string()
             } else {
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}{} {}",
+                "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
                 subtitle,
-                if sel_chat.is_muted() { "🔇" } else { "" },
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
-                match sel_chat.get_profile_image(&context).await? {
+                match sel_chat.get_profile_image(&context).await {
                     Some(icon) => match icon.to_str() {
                         Some(icon) => format!(" Icon: {}", icon),
                         _ => " Icon: Err".to_string(),
                     },
                     _ => "".to_string(),
                 },
-                if sel_chat.is_protected() {
-                    "🛡️"
-                } else {
-                    ""
-                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -679,48 +632,47 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!(
                 "{} messages.",
-                sel_chat.get_id().get_msg_cnt(&context).await?
+                sel_chat.get_id().get_msg_cnt(&context).await
             );
-
-            let time_noticed_start = std::time::SystemTime::now();
             chat::marknoticed_chat(&context, sel_chat.get_id()).await?;
-            let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
-
-            println!(
-                "{:?} to create this list, {:?} to mark all messages as noticed.",
-                time_needed, time_noticed_needed
-            );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: u32 = arg1.parse()?;
-            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
+            let contact_id: libc::c_int = arg1.parse()?;
+            let chat_id = chat::create_by_contact_id(&context, contact_id as u32).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
+        "createchatbymsg" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            let chat_id = chat::create_by_msg_id(&context, msg_id).await?;
+            let chat = Chat::load_from_db(&context, chat_id).await?;
+
+            println!("{}#{} created successfully.", chat_prefix(&chat), chat_id,);
+        }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
+                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createprotected" => {
+        "createverified" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id =
-                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
+            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
 
-            println!("Group#{} created and protected successfully.", chat_id);
+            println!("VerifiedGroup#{} created successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: u32 = arg1.parse()?;
+            let contact_id_0: libc::c_int = arg1.parse()?;
             if chat::add_contact_to_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_0,
+                contact_id_0 as u32,
             )
             .await
             {
@@ -732,11 +684,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: u32 = arg1.parse()?;
+            let contact_id_1: libc::c_int = arg1.parse()?;
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                contact_id_1,
+                contact_id_1 as u32,
             )
             .await?;
 
@@ -762,7 +714,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts =
-                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+                chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await;
             println!("Memberlist:");
 
             log_contactlist(&context, &contacts).await;
@@ -771,7 +723,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 contacts.len(),
                 location::is_sending_locations_to_chat(
                     &context,
-                    Some(sel_chat.as_ref().unwrap().get_id())
+                    sel_chat.as_ref().unwrap().get_id()
                 )
                 .await,
             );
@@ -779,15 +731,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "getlocations" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
-            let contact_id: Option<u32> = arg1.parse().ok();
+            let contact_id = arg1.parse().unwrap_or_default();
             let locations = location::get_range(
                 &context,
-                Some(sel_chat.as_ref().unwrap().get_id()),
+                sel_chat.as_ref().unwrap().get_id(),
                 contact_id,
                 0,
                 0,
             )
-            .await?;
+            .await;
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
@@ -855,14 +807,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), "".into()).await?;
         }
-        "sendimage" | "sendsticker" | "sendfile" => {
+        "sendimage" | "sendfile" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No file given.");
 
             let mut msg = Message::new(if arg0 == "sendimage" {
                 Viewtype::Image
-            } else if arg0 == "sendsticker" {
-                Viewtype::Sticker
             } else {
                 Viewtype::File
             });
@@ -872,22 +822,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
-        "sendhtml" => {
-            ensure!(sel_chat.is_some(), "No chat selected.");
-            ensure!(!arg1.is_empty(), "No html-file given.");
-            let path: &Path = arg1.as_ref();
-            let html = &*fs::read(&path)?;
-            let html = String::from_utf8_lossy(html);
-
-            let mut msg = Message::new(Viewtype::Text);
-            msg.set_html(Some(html.to_string()));
-            msg.set_text(Some(if arg2.is_empty() {
-                path.file_name().unwrap().to_string_lossy().to_string()
-            } else {
-                arg2.to_string()
-            }));
-            chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
-        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -896,18 +830,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
             let chat = if let Some(ref sel_chat) = sel_chat {
-                Some(sel_chat.get_id())
+                sel_chat.get_id()
             } else {
-                None
+                ChatId::new(0)
             };
 
-            let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
-            let time_needed = time_start.elapsed().unwrap_or_default();
+            let msglist = context.search_msgs(chat, arg1).await;
 
             log_msglist(&context, &msglist).await?;
             println!("{} messages.", msglist.len());
-            println!("{:?} to create this list", time_needed);
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -920,7 +851,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, Some(&mut draft))
-                    .await?;
+                    .await;
                 println!("Draft saved.");
             } else {
                 sel_chat
@@ -928,7 +859,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     .unwrap()
                     .get_id()
                     .set_draft(&context, None)
-                    .await?;
+                    .await;
                 println!("Draft deleted.");
             }
         }
@@ -941,6 +872,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
+        "updatedevicechats" => {
+            context.update_device_chats().await?;
+        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -951,7 +885,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Viewtype::Gif,
                 Viewtype::Video,
             )
-            .await?;
+            .await;
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
@@ -972,39 +906,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => unreachable!("arg0={:?}", arg0),
-                    },
-                )
-                .await?;
-        }
-        "mute" | "unmute" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            let duration = match arg0 {
-                "mute" => {
-                    if arg2.is_empty() {
-                        MuteDuration::Forever
-                    } else {
-                        SystemTime::now()
-                            .checked_add(Duration::from_secs(arg2.parse()?))
-                            .map_or(MuteDuration::Forever, MuteDuration::Until)
-                    }
-                }
-                "unmute" => MuteDuration::NotMuted,
-                _ => unreachable!("arg0={:?}", arg0),
-            };
-            chat::set_muted(&context, chat_id, duration).await?;
-        }
-        "protect" | "unprotect" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id
-                .set_protection(
-                    &context,
-                    match arg0 {
-                        "protect" => ProtectionStatus::Protected,
-                        "unprotect" => ProtectionStatus::Unprotected,
-                        _ => unreachable!("arg0={:?}", arg0),
+                        _ => panic!("Unexpected command (This should never happen)"),
                     },
                 )
                 .await?;
@@ -1014,34 +916,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
-        "accept" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.accept(&context).await?;
-        }
-        "blockchat" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.block(&context).await?;
-        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            let res = message::get_msg_info(&context, id).await?;
+            let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
-        "html" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            let file = dirs::home_dir()
-                .unwrap_or_default()
-                .join(format!("msg-{}.html", id.to_u32()));
-            let html = id.get_html(&context).await?.unwrap_or_default();
-            fs::write(&file, html)?;
-            println!("HTML written to: {:#?}", file);
-        }
         "listfresh" => {
-            let msglist = context.get_fresh_msgs().await?;
+            let msglist = context.get_fresh_msgs().await;
 
             log_msglist(&context, &msglist).await?;
             print!("{} fresh messages.", msglist.len());
@@ -1061,7 +943,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await?;
+            message::markseen_msgs(&context, msg_ids).await;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -1073,9 +955,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contacts = Contact::get_all(
                 &context,
                 if arg0 == "listverified" {
-                    DC_GCL_VERIFIED_ONLY | DC_GCL_ADD_SELF
+                    0x1 | 0x2
                 } else {
-                    DC_GCL_ADD_SELF
+                    0x2
                 },
                 Some(arg1),
             )
@@ -1088,7 +970,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, &book).await?;
+                Contact::add_address_book(&context, book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }
@@ -1096,14 +978,14 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = arg1.parse()?;
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
             let mut res = format!(
                 "Contact info for: {}:\nIcon: {}\n",
                 name_n_addr,
-                match contact.get_profile_image(&context).await? {
+                match contact.get_profile_image(&context).await {
                     Some(image) => image.to_str().unwrap().to_string(),
                     None => "NoIcon".to_string(),
                 }
@@ -1133,21 +1015,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
             Contact::delete(&context, arg1.parse()?).await?;
         }
-        "block" => {
-            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
-            Contact::block(&context, contact_id).await?;
-        }
-        "unblock" => {
-            ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
-            Contact::unblock(&context, contact_id).await?;
-        }
-        "listblocked" => {
-            let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await;
-            println!("{} blocked contacts.", contacts.len());
-        }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let res = check_qr(&context, arg1).await;
@@ -1168,10 +1035,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            let socks5_enabled = context
-                .get_config_bool(config::Config::Socks5Enabled)
-                .await?;
-            match provider::get_provider_info(arg1, socks5_enabled).await {
+            match provider::get_provider_info(arg1) {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);
@@ -1195,7 +1059,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r,
+        //         event, event as usize, r as libc::c_int,
         //     );
         // }
         "fileinfo" => {

examples/repl/main.rs

@@ -26,9 +26,8 @@ use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
@@ -57,6 +56,9 @@ fn receive_event(event: EventType) {
         EventType::Error(msg) => {
             error!("{}", msg);
         }
+        EventType::ErrorNetwork(msg) => {
+            error!("[NETWORK] msg={}", msg);
+        }
         EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
@@ -154,7 +156,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
     "set",
     "get",
@@ -162,16 +164,16 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 33] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "createchatbymsg",
     "creategroup",
     "createverified",
     "addmember",
@@ -186,7 +188,6 @@ const CHAT_COMMANDS: [&str; 33] = [
     "send",
     "sendimage",
     "sendfile",
-    "sendhtml",
     "videochat",
     "draft",
     "listmedia",
@@ -194,32 +195,25 @@ const CHAT_COMMANDS: [&str; 33] = [
     "unarchive",
     "pin",
     "unpin",
-    "mute",
-    "unmute",
-    "protect",
-    "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 6] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
+    "star",
+    "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 9] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
     "contactinfo",
     "delcontact",
     "cleanupcontacts",
-    "block",
-    "unblock",
-    "listblocked",
 ];
 const MISC_COMMANDS: [&str; 10] = [
     "getqr",
@@ -235,9 +229,7 @@ const MISC_COMMANDS: [&str; 10] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -259,10 +251,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -283,7 +276,6 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
@@ -308,7 +300,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
+    let (reader_s, reader_r) = async_std::sync::channel(100);
     let input_loop = async_std::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
@@ -317,8 +309,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
@@ -331,7 +323,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
                     rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    async_std::task::block_on(reader_s.send(line));
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
@@ -390,9 +382,9 @@ async fn handle_cmd(
             ctx.configure().await?;
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
+            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -408,8 +400,9 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
+            if let Some(mut qr) =
+                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+            {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")

examples/simple.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
@@ -19,7 +19,7 @@ fn cb(event: EventType) {
         EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

python/CHANGELOG

@@ -1,10 +1,3 @@
-1.51.0
-------
-
-- adapt python bindings and APIs to core51 release 
-  (see CHANGELOG of https://github.com/deltachat/deltachat-core-rust/blob/1.51.0/CHANGELOG.md#1510
-  for more details on all core changes) 
-
 1.44.0
 ------
 

python/README.rst

@@ -11,10 +11,10 @@ Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -58,13 +58,12 @@ end-to-end tests that require accounts on real e-mail servers.
 running "live" tests with temporary accounts
 ---------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
 
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
 One hour is enough to invoke pytest and run all offline and online tests:
 
     pytest
@@ -143,7 +142,7 @@ This docker image can be used to run tests and build Python wheels for all inter
 
     $ docker run -e DCC_NEW_TMP_EMAIL \
        --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image
@@ -152,9 +151,9 @@ Optionally build your own docker image
 If you want to build your own custom docker image you can do this::
 
    $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
+   $ docker build -t deltachat/coredeps ci_scripts/docker_coredeps
 
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
+This will use the ``ci_scripts/docker_coredeps/Dockerfile`` to build
 up docker image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 

python/install_python_bindings.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 """
  setup a python binding development in-place install with cargo debug symbols.
@@ -19,7 +19,8 @@ if __name__ == "__main__":
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
     if target == 'release':
-        os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
+        extra = " -C lto=on -C embed-bitcode=yes"
+        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
         cmd.append("--release")
 
     print("running:", " ".join(cmd))

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient', 'requests'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -77,7 +77,6 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
         configtracker = ac.configure()
         configtracker.wait_finish()
 

python/src/deltachat/_build.py

@@ -9,6 +9,8 @@ import subprocess
 import tempfile
 import textwrap
 import types
+from os.path import abspath
+from os.path import dirname as dn
 
 import cffi
 
@@ -48,7 +50,6 @@ def system_build_flags():
     flags.objs = []
     flags.incs = []
     flags.extra_link_args = []
-    return flags
 
 
 def extract_functions(flags):
@@ -144,13 +145,9 @@ def extract_defines(flags):
                 | DC_STR
                 | DC_CONTACT_ID
                 | DC_GCL
-                | DC_GCM
-                | DC_SOCKET
                 | DC_CHAT
                 | DC_PROVIDER
                 | DC_KEY_GEN
-                | DC_IMEX
-                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -168,8 +165,11 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()

python/src/deltachat/account.py

@@ -89,22 +89,6 @@ class Account(object):
             d[key.lower()] = value
         return d
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        log("=============== " + self.get_config("displayname") + " ===============")
-        cursor = 0
-        for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
     def set_stock_translation(self, id, string):
         """ set stock translation string.
 
@@ -230,19 +214,6 @@ class Account(object):
         :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        return Contact(self, contact_id)
-
-    def get_contact(self, obj):
-        if isinstance(obj, Contact):
-            return obj
-        (_, addr) = self.get_contact_addr_and_name(obj)
-        return self.get_contact_by_addr(addr)
-
-    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -258,7 +229,13 @@ class Account(object):
 
         if name is None and displayname:
             name = displayname
-        return (name, addr)
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
+        addr = as_dc_charpointer(addr)
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        return Contact(self, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -286,17 +263,6 @@ class Account(object):
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self):
-        """ return a list of all blocked contacts.
-
-        :returns: list of :class:`deltachat.contact.Contact` objects.
-        """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -330,6 +296,9 @@ class Account(object):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+
     def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
@@ -364,8 +333,8 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self):
-        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
         """ return Message instance.
@@ -422,23 +391,23 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
+        return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
 
-        Note that the account has to be stopped; call stop_io() if necessary.
+        Note that the account does not have to be started.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
     def _export(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
@@ -448,7 +417,7 @@ class Account(object):
 
         Note that the account does not have to be started.
         """
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
+        self._import(path, imex_cmd=2)
 
     def import_all(self, path):
         """import delta chat state from the specified backup `path` (a file).
@@ -456,22 +425,21 @@ class Account(object):
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
+        if not self.is_started():
+            raise RuntimeError("IO not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -568,15 +536,6 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self):
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 
@@ -599,34 +558,12 @@ class Account(object):
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
-    def maybe_network(self):
-        """This function should be called when there is a hint
-        that the network is available again,
-        e.g. as a response to system event reporting network availability.
-        The library will try to send pending messages out immediately.
-
-        Moreover, to have a reliable state
-        when the app comes to foreground with network available,
-        it may be reasonable to call the function also at that moment.
-
-        It is okay to call the function unconditionally when there is
-        network available, however, calling the function
-        _without_ having network may interfere with the backoff algorithm
-        and will led to let the jobs fail faster, with fewer retries
-        and may avoid messages being sent out.
-
-        Finally, if the context was created by the dc_accounts_t account manager
-        (currently not implemented in the Python bindings),
-        use dc_accounts_maybe_network() instead of this function
-        """
-        lib.dc_maybe_network(self._dc_context)
-
-    def configure(self, reconfigure=False):
+    def configure(self):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
-        assert self.is_configured() == reconfigure
+        assert not self.is_configured()
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)
@@ -634,6 +571,9 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
+    def is_started(self):
+        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
+
     def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
@@ -643,8 +583,11 @@ class Account(object):
         self.log("stop_ongoing")
         self.stop_ongoing()
 
-        self.log("dc_stop_io (stop core IO scheduler)")
-        lib.dc_stop_io(self._dc_context)
+        if bool(lib.dc_is_io_running(self._dc_context)):
+            self.log("dc_stop_io (stop core IO scheduler)")
+            lib.dc_stop_io(self._dc_context)
+        else:
+            self.log("stop_scheduler called on non-running context")
 
     def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove

python/src/deltachat/chat.py

@@ -50,14 +50,6 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self):
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self):
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self):
@@ -65,7 +57,17 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
+
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
 
     def is_muted(self):
         """ return true if this chat is muted.
@@ -74,13 +76,6 @@ class Chat(object):
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -90,20 +85,12 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self):
-        """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+    def is_verified(self):
+        """ return True if this chat is a verified group.
 
-        :returns: True if the chat is writable, False otherwise
+        :returns: True if chat is verified, False otherwise.
         """
-        return lib.dc_chat_can_send(self._dc_chat)
-
-    def is_protected(self):
-        """ return True if this chat is a protected chat.
-
-        :returns: True if chat is protected, False otherwise.
-        """
-        return lib.dc_chat_is_protected(self._dc_chat)
+        return lib.dc_chat_is_verified(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.
@@ -175,13 +162,6 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self):
-        """Return encryption info for this chat.
-
-        :returns: a string with encryption preferences of all chat members"""
-        res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
-        return from_dc_charpointer(res)
-
     def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
@@ -262,19 +242,17 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """ create a new prepared message.
 
         :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
+        :returns: :class:`deltachat.message.Message` instance.
         """
         msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
+        # invalidate passed in message which is not safe to use anymore
+        msg._dc_msg = msg.id = None
+        return Message.from_db(self.account, msg_id)
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
         """ prepare a message for sending and return the resulting Message instance.
@@ -388,7 +366,7 @@ class Chat(object):
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.get_contact(obj)
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
@@ -512,23 +490,18 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.utcfromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i)
-                ),
-                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
-            )
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp):
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
         self.accuracy = accuracy
         self.timestamp = timestamp
-        self.marker = marker
 
     def __eq__(self, other):
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -52,17 +52,9 @@ class Contact(object):
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        """ Block or unblock a contact. """
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
-    def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
-        return lib.dc_block_contact(self.account._dc_context, self.id, True)
-
-    def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
-        return lib.dc_block_contact(self.account._dc_context, self.id, False)
-
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
@@ -77,14 +69,6 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    @property
-    def status(self):
-        """Get contact status.
-
-        :returns: contact status, empty string if it doesn't exist.
-        """
-        return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
-
     def create_chat(self):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 

python/src/deltachat/direct_imap.py

@@ -9,9 +9,7 @@ import ssl
 import pathlib
 from imapclient import IMAPClient
 from imapclient.exceptions import IMAPClientError
-import imaplib
 import deltachat
-from deltachat import const
 
 
 SEEN = b'\\Seen'
@@ -26,30 +24,13 @@ def dc_account_extra_configure(account):
     """ Reset the account (we reuse accounts across tests)
     and make 'account.direct_imap' available for direct IMAP ops.
     """
-    try:
-
-        if not hasattr(account, "direct_imap"):
-            imap = DirectImap(account)
-
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete(ALL, expunge=True)
-                else:
-                    imap.conn.delete_folder(folder)
-                    # We just deleted the folder, so we have to make DC forget about it, too
-                    if account.get_config("configured_sentbox_folder") == folder:
-                        account.set_config("configured_sentbox_folder", None)
-                    if account.get_config("configured_spam_folder") == folder:
-                        account.set_config("configured_spam_folder", None)
-
-            setattr(account, "direct_imap", imap)
-
-    except Exception as e:
-        # Uncaught exceptions here would lead to a timeout without any note written to the log
-        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
-        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
-        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
+        imap.delete(ALL, expunge=True)
+        setattr(account, "direct_imap", imap)
 
 
 @deltachat.global_hookimpl
@@ -69,31 +50,18 @@ class DirectImap:
         self.connect()
 
     def connect(self):
-        host = self.account.get_config("configured_mail_server")
-        port = int(self.account.get_config("configured_mail_port"))
-        security = int(self.account.get_config("configured_mail_security"))
+        ssl_context = ssl.create_default_context()
 
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
         user = self.account.get_config("addr")
         pw = self.account.get_config("mail_pw")
-
-        if security == const.DC_SOCKET_PLAIN:
-            ssl_context = None
-        else:
-            ssl_context = ssl.create_default_context()
-
-            # don't check if certificate hostname doesn't match target hostname
-            ssl_context.check_hostname = False
-
-            # don't check if the certificate is trusted by a certificate authority
-            ssl_context.verify_mode = ssl.CERT_NONE
-
-        if security == const.DC_SOCKET_STARTTLS:
-            self.conn = IMAPClient(host, port, ssl=False)
-            self.conn.starttls(ssl_context)
-        elif security == const.DC_SOCKET_PLAIN:
-            self.conn = IMAPClient(host, port, ssl=False)
-        elif security == const.DC_SOCKET_SSL:
-            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
@@ -108,12 +76,6 @@ class DirectImap:
         except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
-    def create_folder(self, foldername):
-        try:
-            self.conn.create_folder(foldername)
-        except imaplib.IMAP4.error as e:
-            print("Can't create", foldername, "probably it already exists:", str(e))
-
     def select_folder(self, foldername):
         assert not self._idling
         return self.conn.select_folder(foldername)
@@ -173,6 +135,21 @@ class DirectImap:
     def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
     def dump_imap_structures(self, dir, logfile):
         assert not self._idling
         stream = io.StringIO()
@@ -249,18 +226,3 @@ class DirectImap:
             res = self.conn.idle_done()
             self._idling = False
             return res
-
-    def append(self, folder, msg):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
-        """
-        if msg.startswith("\n"):
-            msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(folder, msg)
-
-    def get_uid_by_message_id(self, message_id):
-        msgs = self.conn.search(['HEADER', 'MESSAGE-ID', message_id])
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]

python/src/deltachat/events.py

@@ -103,41 +103,6 @@ class FFIEventTracker:
             if rex.search(ev.data2):
                 return ev
 
-    def get_info_regex_groups(self, regex, check_error=True):
-        rex = re.compile(regex)
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
-            m = rex.match(ev.data2)
-            if m is not None:
-                return m.groups()
-
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):

python/src/deltachat/message.py

@@ -21,8 +21,8 @@ class Message(object):
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
-        msg_id = self.id
-        assert msg_id is not None and msg_id >= 0, repr(msg_id)
+        self.id = lib.dc_msg_get_id(dc_msg)
+        assert self.id is not None and self.id >= 0, repr(self.id)
 
     def __eq__(self, other):
         return self.account == other.account and self.id == other.id
@@ -46,13 +46,9 @@ class Message(object):
     def new_empty(cls, account, view_type):
         """ create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param: view_type is "text", "audio", "video", "file"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = get_viewtype_code_from_name(view_type)
         return Message(account, ffi.gc(
             lib.dc_msg_new(account._dc_context, view_type_code),
             lib.dc_msg_unref
@@ -61,18 +57,16 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a contact request
+        If the message is a deaddrop contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.chat.accept()
-        return self.chat
-
-    @props.with_doc
-    def id(self):
-        """id of this message. """
-        return lib.dc_msg_get_id(self._dc_msg)
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):
@@ -83,23 +77,6 @@ class Message(object):
         """set text of this message. """
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
-    @props.with_doc
-    def html(self):
-        """html text of this messages (might be empty if not an html message). """
-        return from_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
-
-    def has_html(self):
-        """return True if this message has an html part, False otherwise."""
-        return lib.dc_msg_has_html(self._dc_msg)
-
-    def set_html(self, html_text):
-        """set the html part of this message.
-
-        It is possible to have text and html part at the same time.
-        """
-        lib.dc_msg_set_html(self._dc_msg, as_dc_charpointer(html_text))
-
     @props.with_doc
     def filename(self):
         """filename if there was an attachment, otherwise empty string. """
@@ -138,10 +115,6 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
@@ -239,11 +212,6 @@ class Message(object):
                 return email.message_from_bytes(s)
             return email.message_from_string(s)
 
-    @property
-    def error(self):
-        """Error message"""
-        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
-
     @property
     def chat(self):
         """chat this message was posted in.
@@ -254,20 +222,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    @props.with_doc
-    def override_sender_name(self):
-        """the name that should be shown over the message instead of the contact display name.
-
-        Usually used to impersonate someone else.
-        """
-        return from_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
-
-    def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
-
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
 
@@ -380,10 +334,6 @@ class Message(object):
         """ return True if it's a gif message. """
         return self._view_type == const.DC_MSG_GIF
 
-    def is_sticker(self):
-        """ return True if it's a sticker message. """
-        return self._view_type == const.DC_MSG_STICKER
-
     def is_audio(self):
         """ return True if it's an audio message. """
         return self._view_type == const.DC_MSG_AUDIO
@@ -404,22 +354,21 @@ class Message(object):
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    const.DC_MSG_TEXT: 'text',
+    const.DC_MSG_IMAGE: 'image',
+    const.DC_MSG_GIF: 'gif',
+    const.DC_MSG_AUDIO: 'audio',
+    const.DC_MSG_VIDEO: 'video',
+    const.DC_MSG_FILE: 'file'
 }
 
 
 def get_viewtype_code_from_name(view_type_name):
-    code = _view_type_mapping.get(view_type_name)
-    if code is not None:
-        return code
+    for code, value in _view_type_mapping.items():
+        if value == view_type_name:
+            return code
     raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+                     "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
 
 
 #

python/src/deltachat/testplugin.py

@@ -312,31 +312,24 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io([ac1])
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
             ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io([ac1, ac2])
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
         def get_many_online_accounts(self, num, move=True):
             accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
-            self.wait_configure_and_start_io(accounts)
+            self.wait_configure_and_start_io()
             for acc in accounts:
                 acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
-            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
-            direct_imap object of an online account. This simulates the user setting
-            up a new device without importing a backup.
-
-            `pre_generated_key` only means that a key from python/tests/data/key is
-            used in order to speed things up.
-            """
             self.live_count += 1
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
@@ -356,29 +349,21 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac._configtracker = ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self, accounts=None):
-            if accounts is None:
-                accounts = self._accounts[:]
+        def wait_configure_and_start_io(self):
             started_accounts = []
-            for acc in accounts:
-                if acc not in started_accounts:
-                    self.wait_configure(acc)
-                    acc.set_config("bcc_self", "0")
-                    if acc.is_configured():
-                        acc.start_io()
-                        started_accounts.append(acc)
-                    print("{}: {} account was started".format(
-                        acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                    started_accounts.append(acc)
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
             for acc in started_accounts:
                 acc._evtracker.wait_all_initial_fetches()
 
-        def wait_configure(self, acc):
-            if hasattr(acc, "_configtracker"):
-                acc._configtracker.wait_finish()
-                acc._evtracker.consume_events()
-                acc.get_device_chat().mark_noticed()
-                del acc._configtracker
-
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
@@ -414,13 +399,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
         def dump_imap_summary(self, logfile):
             for ac in self._accounts:
-                ac.dump_account_info(logfile=logfile)
                 imap = getattr(ac, "direct_imap", None)
                 if imap is not None:
                     try:
                         imap.idle_done()
                     except Exception:
                         pass
+                    imap.dump_account_info(logfile=logfile)
                     imap.dump_imap_structures(tmpdir, logfile=logfile)
 
         def get_accepted_chat(self, ac1, ac2):
@@ -516,9 +501,6 @@ def lp():
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg):
-            print("  " + msg)
-
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -20,16 +20,6 @@ class ImexTracker:
         elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
             self._imex_events.put(ffi_event.data2)
 
-    def wait_progress(self, target_progress, progress_upper_limit=1000, progress_timeout=60):
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
-                    str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
-                return ev
-            if ev == 0:
-                return None
-
     def wait_finish(self, progress_timeout=60):
         """ Return list of written files, raise ValueError if ExportFailed. """
         files_written = []

python/tests/auditwheels.py

@@ -1,17 +1,15 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                 "--plat", "manylinux2014_x86_64"])

python/tox.ini

@@ -1,6 +1,7 @@
 [tox]
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py37
     lint
     auditwheels
 
@@ -12,6 +13,7 @@ passenv =
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
+    DCC_PY_LIVECONFIG
     DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
@@ -45,9 +47,8 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    # Pin dependencies to the versions which actually work with Python 3.5.
-    sphinx==3.4.3
-    breathe==4.28.0
+    sphinx
+    breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

run-integration-tests.sh

@@ -4,11 +4,11 @@
 # purposes.  Any arguments are passed straight to tox.  E.g. to run
 # only one environment run with:
 #
-#   scripts/run-integration-tests.sh -e py35
+#   ./run-integration-tests.sh -e py35
 #
 # To also run with `pytest -x` use:
 #
-#   scripts/run-integration-tests.sh -e py35 -- -x
+#   ./run-integration-tests.sh -e py35 -- -x
 
 export DCC_RS_DEV=$(pwd)
 export DCC_RS_TARGET=${DCC_RS_TARGET:-release}
@@ -23,6 +23,9 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
+    export DCC_PY_LIVECONFIG=liveconfig
+fi
 tox "$@"
 ret=$?
 popd

rust-toolchain

@@ -1 +1 @@
-1.54.0
+1.45.0

scripts/concourse/README.md

@@ -1,21 +0,0 @@
-# Concourse CI pipeline
-
-`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
-that builds C documentation, Python documentation, Python wheels for `x86_64`
-and `aarch64` and Python source packages, and uploads them.
-
-To setup the pipeline run
-```
-fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
-```
-where `secret.yml` contains the following secrets:
-```
-c.delta.chat:
-  private_key: |
-    -----BEGIN RSA PRIVATE KEY-----
-    ...
-    -----END RSA PRIVATE KEY-----
-devpi:
-  login: dc
-  password: ...
-```

scripts/concourse/docs_wheels.yml

@@ -1,232 +0,0 @@
-resources:
-- name: deltachat-core-rust
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-
-- name: deltachat-core-rust-release
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-    tag_filter: "py-*"
-
-jobs:
-- name: doxygen
-  plan:
-  - get: deltachat-core-rust
-    trigger: true
-
-  # Build Doxygen documentation
-  - task: build-doxygen
-    config:
-      inputs:
-        - name: deltachat-core-rust
-      outputs:
-        - name: c-docs
-      image_resource:
-        source:
-          repository: hrektts/doxygen
-        type: registry-image
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
-          cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
-
-  - task: upload-c-docs
-    config:
-      inputs:
-        - name: c-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
-- name: python-x86_64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-docs
-          path: ./python/doc/_build/
-        # Source packages
-        - name: py-dist
-          path: ./python/.docker-tox/dist/
-        # Binary wheels
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload python docs to py.delta.chat
-  - task: upload-py-docs
-    config:
-      inputs:
-        - name: py-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
-  # Upload x86_64 wheels and source packages
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-        - name: py-dist
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*
-          devpi upload py-dist/*
-
-- name: python-aarch64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload aarch64 wheels
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*

scripts/coverage.sh

@@ -1,26 +0,0 @@
-#!/bin/sh
-set -eu
-
-if ! which grcov 2>/dev/null 1>&2; then
-    echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
-    echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
-    exit 1
-fi
-
-# Allow `-Z` flags without using nightly Rust.
-export RUSTC_BOOTSTRAP=1
-
-# We are using `-Zprofile` instead of source-based coverage [1]
-# (`-Zinstrument-coverage`) due to a bug resulting in empty reports [2].
-#
-# [1] https://blog.rust-lang.org/inside-rust/2020/11/12/source-based-code-coverage.html
-# [2] https://github.com/mozilla/grcov/issues/595
-
-export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
-export RUSTDOCFLAGS="-Cpanic=abort"
-cargo clean
-cargo build
-cargo test
-
-grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./coverage/

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.6 environment as the base environment 
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
-
-popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.6 environment as the base environment
-/opt/python/cp36-cp36m/bin/pip install tox devpi-client auditwheel
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.6.*/bin/tox
-ln -s /opt/_internal/cpython-3.6.*/bin/devpi
-ln -s /opt/_internal/cpython-3.6.*/bin/auditwheel
-
-popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/manual_remote_tests.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -xe
-
-JOB=${1:?need to specify 'rust' or 'python'}
-BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
-REPONAME="$(basename $(git rev-parse --show-toplevel))"
-
-time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"

scripts/remote_tests_rust.sh

@@ -1,29 +0,0 @@
-#!/bin/bash 
-
-BUILD_ID=${1:?specify build ID}
-
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
-
-set -e
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-echo "--- Running Rust tests remotely"
-
-ssh $SSHTARGET <<_HERE
-    set +x -e
-    # make sure all processes exit when ssh dies
-    shopt -s huponexit
-    export RUSTC_WRAPPER=\`which sccache\`
-    cd $BUILDDIR
-    export TARGET=x86_64-unknown-linux-gnu
-    export RUSTC_WRAPPER=sccache
-
-    bash scripts/run-rust-test.sh
-_HERE
-

scripts/run-doxygen.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-
-cd deltachat-ffi 
-PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

set_core_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 
 import os
 import sys
@@ -63,14 +63,13 @@ def main():
     ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
-    if "alpha" not in newversion:
-        for line in open("CHANGELOG.md"):
-            ## 1.25.0
-            if line.startswith("## "):
-                if line[2:].strip().startswith(newversion):
-                    break
-        else:
-            raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+    for line in open("CHANGELOG.md"):
+        ## 1.25.0
+        if line.startswith("## "):
+            if line[2:].strip().startswith(newversion):
+                break
+    else:
+        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
@@ -82,10 +81,9 @@ def main():
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit, on master make sure to: ")
+    print("after commit make sure to: ")
     print("")
-    print("   git tag -a {}".format(newversion))
-    print("   git push origin {}".format(newversion))
+    print("   git tag {}".format(newversion))
     print("")
 
 

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.33.0
+Version: 0.32.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -32,7 +32,7 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
@@ -301,9 +301,9 @@ to add a `Chat-Group-Avatar` only on image changes.
 
 A user MAY have a profile-image that MAY be distributed to their contacts.
 To change or set the profile-image,
-the messenger MUST add the header `Chat-User-Avatar: base64:IMAGEDATA`.
-To bypass limits of headers, it is recommended not to use the outer header
-and to limit the size to 20k.
+the messenger MUST attach an image file to a message
+and MUST add the header `Chat-User-Avatar`
+with the value set to the image name.
 
 To remove the profile-image,
 the messenger MUST add the header `Chat-User-Avatar: 0`.
@@ -320,14 +320,19 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     From: sender@domain
     To: rcpt@domain
     Chat-Version: 1.0
+    Chat-User-Avatar: photo.jpg
     Subject: Chat: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 
     --==break==
     Content-Type: text/plain
-    Chat-User-Avatar: base64:AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQY ...
 
     Hello, I've changed my profile image.
+    --==break==
+    Content-Type: image/jpeg
+    Content-Disposition: attachment; filename="photo.jpg"
+
+    AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQYFBA ...
     --==break==--
 
 The image format SHOULD be image/jpeg or image/png.
@@ -337,11 +342,6 @@ in the same message.
 To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
-In older specs, the profile-image was sent as an attachment
-and `Chat-User-Avatar:` specified its name.
-However, it turned out that these attachements are kind of unuexpected to users,
-therefore the profile-image go to the header now.
-
 
 # Locations
 
@@ -401,41 +401,9 @@ it is fine if the location is detected on forwarding etc.
     </kml>
 
 
-# Stickers
+# Miscellaneous
 
-Stickers are send as normal images
-with the additional header `Chat-Content: sticker`.
-
-It is discouraged to send stickers together with user generated text,
-however, stickers can be used as a reply to a message
-and also the footer should be set as usual.
-
-    From: alice@example.org
-    To: bob@example.com
-    Chat-Version: 1.0
-    Chat-Content: sticker
-    Message-ID: Mr.12345uvwxyZ.0005@example.org
-    Subject: Message from Alice
-    Content-Type: multipart/mixed; boundary="==break=="
-
-    --==break==
-    Content-Type: text/plain
-
-    -- 
-    Hi there! I am using this new messenger!
-    --==break==
-    Content-Type: image/png
-    Content-Disposition: attachment; filename="sticker.png"
-
-    R0lGODlhpAGkAfe9AP+zd2eQkZhrI//z9v++PMb///+scrdDT3BtbtrZ2f/LQSsREcdIVf9 ...
-    --==break==--
-
-Typical sticker formats are `image/png`, `image/gif` and `image/webp`.
-Animated stickers are supported
-by just using an image format that supports animation.
-
-
-# Voice messages
+Messengers SHOULD use the header `In-Reply-To` as usual.
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -449,11 +417,6 @@ This allows the receiver to show the time without knowing the file format.
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 
-
-# Miscellaneous
-
-Messengers SHOULD use the header `In-Reply-To` as usual.
-
 Messengers MAY send and receive Message Disposition Notifications
 (MDNs, [RFC 8098](https://tools.ietf.org/html/rfc8098),
 [RFC 3503](https://tools.ietf.org/html/rfc3503))
@@ -474,4 +437,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2021 Delta Chat contributors.
+Copyright © 2017-2020 Delta Chat contributors.

src/accounts.rs

@@ -1,18 +1,16 @@
-//! # Account manager module.
-
 use std::collections::BTreeMap;
 
-use async_std::channel::{Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
 use async_std::sync::{Arc, RwLock};
 use uuid::Uuid;
 
-use anyhow::{ensure, Context as _, Result};
+use anyhow::{ensure, Context as _};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
+use crate::error::Result;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -21,7 +19,6 @@ pub struct Accounts {
     dir: PathBuf,
     config: Config,
     accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
-    emitter: EventEmitter,
 }
 
 impl Accounts {
@@ -34,13 +31,19 @@ impl Accounts {
         Accounts::open(dir).await
     }
 
-    /// Creates a new default structure.
+    /// Creates a new default structure, including a default account.
     pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(os_name.clone(), dir).await?;
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
 
         Ok(())
     }
@@ -56,16 +59,10 @@ impl Accounts {
         let config = Config::from_file(config_file).await?;
         let accounts = config.load_accounts().await?;
 
-        let emitter = EventEmitter::new();
-        for account in accounts.values() {
-            emitter.add_account(account).await?;
-        }
-
         Ok(Self {
             dir,
             config,
             accounts: Arc::new(RwLock::new(accounts)),
-            emitter,
         })
     }
 
@@ -75,17 +72,14 @@ impl Accounts {
     }
 
     /// Get the currently selected account.
-    pub async fn get_selected_account(&self) -> Option<Context> {
+    pub async fn get_selected_account(&self) -> Context {
         let id = self.config.get_selected_account().await;
-        self.accounts.read().await.get(&id).cloned()
-    }
-
-    /// Returns the currently selected account's id or None if no account is selected.
-    pub async fn get_selected_account_id(&self) -> Option<u32> {
-        match self.config.get_selected_account().await {
-            0 => None,
-            id => Some(id),
-        }
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
     }
 
     /// Select the given account.
@@ -101,7 +95,6 @@ impl Accounts {
         let account_config = self.config.new_account(&self.dir).await?;
 
         let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
         self.accounts.write().await.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -128,7 +121,6 @@ impl Accounts {
     /// Migrate an existing account into this structure.
     pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
-        let walfile = Context::derive_walfile(&dbfile);
 
         ensure!(
             dbfile.exists().await,
@@ -144,31 +136,15 @@ impl Accounts {
         let old_id = self.config.get_selected_account().await;
 
         // create new account
-        let account_config = self
-            .config
-            .new_account(&self.dir)
-            .await
-            .context("failed to create new account")?;
+        let account_config = self.config.new_account(&self.dir).await?;
 
         let new_dbfile = account_config.dbfile().into();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
-        let new_walfile = Context::derive_walfile(&new_dbfile);
 
         let res = {
-            fs::create_dir_all(&account_config.dir)
-                .await
-                .context("failed to create dir")?;
-            fs::rename(&dbfile, &new_dbfile)
-                .await
-                .context("failed to rename dbfile")?;
-            fs::rename(&blobdir, &new_blobdir)
-                .await
-                .context("failed to rename blobdir")?;
-            if walfile.exists().await {
-                fs::rename(&walfile, &new_walfile)
-                    .await
-                    .context("failed to rename walfile")?;
-            }
+            fs::create_dir_all(&account_config.dir).await?;
+            fs::rename(&dbfile, &new_dbfile).await?;
+            fs::rename(&blobdir, &new_blobdir).await?;
             Ok(())
         };
 
@@ -181,7 +157,6 @@ impl Accounts {
                     account_config.id,
                 )
                 .await?;
-                self.emitter.add_account(&ctx).await?;
                 self.accounts.write().await.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
@@ -206,23 +181,23 @@ impl Accounts {
         self.accounts.read().await.keys().copied().collect()
     }
 
-    /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
-    ///
-    /// Returns whether all accounts finished their background work.
-    /// DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
-    ///
-    /// iOS can:
-    /// - call dc_start_io() (in case IO was not running)
-    /// - call dc_maybe_network()
-    /// - while dc_accounts_all_work_done() returns false:
-    ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
-    pub async fn all_work_done(&self) -> bool {
-        for account in self.accounts.read().await.values() {
-            if !account.all_work_done().await {
-                return false;
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
             }
         }
-        true
     }
 
     pub async fn start_io(&self) {
@@ -246,70 +221,32 @@ impl Accounts {
         }
     }
 
-    pub async fn maybe_network_lost(&self) {
-        let accounts = &*self.accounts.read().await;
-        for account in accounts.values() {
-            account.maybe_network_lost().await;
-        }
-    }
-
-    /// Returns unified event emitter.
+    /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        self.emitter.clone()
+        let emitters: Vec<_> = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(_id, a)| a.get_event_emitter())
+            .collect();
+
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
-/// Unified event emitter for multiple accounts.
-#[derive(Debug, Clone)]
-pub struct EventEmitter {
-    /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<crate::events::EventEmitter>>>,
-
-    /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<crate::events::EventEmitter>,
-
-    /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<crate::events::EventEmitter>,
-}
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
 
 impl EventEmitter {
-    pub fn new() -> Self {
-        let (sender, receiver) = async_std::channel::unbounded();
-        Self {
-            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
-            sender,
-            receiver,
-        }
-    }
-
     /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
     pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv()).unwrap_or_default()
+        async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
-    pub async fn recv(&mut self) -> Result<Option<Event>> {
-        let mut stream = self.stream.write().await;
-        loop {
-            match futures::future::select(self.receiver.recv(), stream.next()).await {
-                futures::future::Either::Left((emitter, _)) => {
-                    stream.push(emitter?);
-                }
-                futures::future::Either::Right((ev, _)) => return Ok(ev),
-            }
-        }
-    }
-
-    /// Add event emitter of a new account to the aggregate event emitter.
-    pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender.send(context.get_event_emitter()).await?;
-        Ok(())
-    }
-}
-
-impl Default for EventEmitter {
-    fn default() -> Self {
-        Self::new()
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
@@ -320,14 +257,13 @@ impl async_std::stream::Stream for EventEmitter {
         mut self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self).poll_next(cx)
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
     }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
-/// Account manager configuration file.
 #[derive(Debug, Clone)]
 pub struct Config {
     file: PathBuf,
@@ -402,7 +338,7 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
             let inner = &mut self.inner.write().await;
             let id = inner.next_id;
@@ -411,6 +347,7 @@ impl Config {
 
             inner.accounts.push(AccountConfig {
                 id,
+                name: String::new(),
                 dir: target_dir.into(),
                 uuid,
             });
@@ -442,7 +379,7 @@ impl Config {
         self.sync().await
     }
 
-    async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
         self.inner
             .read()
             .await
@@ -473,11 +410,12 @@ impl Config {
     }
 }
 
-/// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-struct AccountConfig {
+pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
+    /// Display name
+    pub name: String,
     /// Root directory for all data for this account.
     pub dir: std::path::PathBuf,
     pub uuid: Uuid,
@@ -500,8 +438,6 @@ mod tests {
         let p: PathBuf = dir.path().join("accounts1").into();
 
         let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        accounts1.add_account().await.unwrap();
-
         let accounts2 = Accounts::open(p).await.unwrap();
 
         assert_eq!(accounts1.accounts.read().await.len(), 1);
@@ -524,11 +460,7 @@ mod tests {
         let p: PathBuf = dir.path().join("accounts").into();
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
 
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, 1);
         assert_eq!(accounts.accounts.read().await.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
@@ -545,35 +477,14 @@ mod tests {
         assert_eq!(accounts.accounts.read().await.len(), 1);
     }
 
-    #[async_std::test]
-    async fn test_accounts_remove_last() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-        assert_eq!(accounts.config.get_selected_account().await, 0);
-
-        let id = accounts.add_account().await?;
-        assert!(accounts.get_selected_account().await.is_some());
-        assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, id);
-
-        accounts.remove_account(id).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-
-        Ok(())
-    }
-
     #[async_std::test]
     async fn test_migrate_account() {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
         let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
@@ -589,16 +500,13 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.read().await.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
 
-        let ctx = accounts.get_selected_account().await.unwrap();
+        let ctx = accounts.get_selected_account().await;
         assert_eq!(
             "me@mail.com",
-            ctx.get_config(crate::config::Config::Addr)
-                .await
-                .unwrap()
-                .unwrap()
+            ctx.get_config(crate::config::Config::Addr).await.unwrap()
         );
     }
 
@@ -610,7 +518,7 @@ mod tests {
 
         let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        for expected_id in 1..10 {
+        for expected_id in 2..10 {
             let id = accounts.add_account().await.unwrap();
             assert_eq!(id, expected_id);
         }
@@ -620,86 +528,4 @@ mod tests {
             assert_eq!(ids.get(i), Some(&expected_id));
         }
     }
-
-    #[async_std::test]
-    async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-        let dummy_accounts = 10;
-
-        let (id0, id1, id2) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
-            accounts.add_account().await?;
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 1);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("one@example.org"))
-                .await?;
-
-            let id1 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id1).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("two@example.org"))
-                .await?;
-
-            // add and remove some accounts and force a gap (ids must not be reused)
-            for _ in 0..dummy_accounts {
-                let to_delete = accounts.add_account().await?;
-                accounts.remove_account(to_delete).await?;
-            }
-
-            let id2 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id2).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("three@example.org"))
-                .await?;
-
-            accounts.select_account(id1).await?;
-
-            (id0, id1, id2)
-        };
-        assert!(id0 > 0);
-        assert!(id1 > id0);
-        assert!(id2 > id1 + dummy_accounts);
-
-        let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
-            let ctx = accounts.get_selected_account().await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 3);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("one@example.org".to_string())
-            );
-
-            let id1 = *ids.get(1).unwrap();
-            let t = accounts.get_account(id1).await.unwrap();
-            assert_eq!(
-                t.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let id2 = *ids.get(2).unwrap();
-            let ctx = accounts.get_account(id2).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("three@example.org".to_string())
-            );
-
-            (id0, id1, id2)
-        };
-        assert_eq!(id0, id0_reopened);
-        assert_eq!(id1, id1_reopened);
-        assert_eq!(id2, id2_reopened);
-
-        Ok(())
-    }
 }

src/aheader.rs

@@ -1,4 +1,4 @@
-//! # Autocrypt header module.
+//! # Autocrypt header module
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
@@ -6,7 +6,7 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use crate::contact::addr_cmp;
+use crate::contact::*;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};

src/blob.rs

@@ -1,6 +1,5 @@
-//! # Blob directory management.
+//! # Blob directory management
 
-use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
 
@@ -8,21 +7,14 @@ use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::format_err;
-use anyhow::Context as _;
-use anyhow::Error;
-use image::DynamicImage;
 use image::GenericImageView;
-use image::ImageFormat;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
-use crate::constants::{
-    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
-    WORSE_IMAGE_SIZE,
-};
+use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::EventType;
 use crate::message;
 
@@ -58,12 +50,12 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: &str,
+        suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -71,12 +63,6 @@ impl<'a> BlobObject<'a> {
                 blobname: name.clone(),
                 cause: err.into(),
             })?;
-
-        // workaround a bug in async-std
-        // (the executor does not handle blocking operation in Drop correctly,
-        // see <https://github.com/async-rs/async-std/issues/900>)
-        let _ = file.flush().await;
-
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
@@ -137,17 +123,18 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: &Path,
+        src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file = fs::File::open(src)
-            .await
-            .map_err(|err| BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: String::from(""),
-                src: src.to_path_buf(),
-                cause: err,
-            })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
+        let mut src_file =
+            fs::File::open(src.as_ref())
+                .await
+                .map_err(|err| BlobError::CopyFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: String::from(""),
+                    src: src.as_ref().to_path_buf(),
+                    cause: err,
+                })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) =
             BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
@@ -160,14 +147,10 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.to_path_buf(),
+                src: src.as_ref().to_path_buf(),
                 cause: err,
             });
         }
-
-        // workaround, see create() for details
-        let _ = dst_file.flush().await;
-
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
@@ -194,13 +177,16 @@ impl<'a> BlobObject<'a> {
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        if src.starts_with(context.get_blobdir()) {
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'_>, BlobError> {
+        if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.starts_with("$BLOBDIR/") {
-            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -221,22 +207,23 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &'a Context,
-        path: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let rel_path =
-            path.strip_prefix(context.get_blobdir())
-                .map_err(|_| BlobError::WrongBlobdir {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    src: path.to_path_buf(),
-                })?;
-        if !BlobObject::is_acceptible_blob_name(rel_path) {
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::WrongBlobdir {
+                blobdir: context.get_blobdir().to_path_buf(),
+                src: path.as_ref().to_path_buf(),
+            })?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.to_path_buf(),
+                blobname: path.as_ref().to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.to_path_buf(),
+            blobname: path.as_ref().to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -380,25 +367,26 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
+    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
 
-        let img_wh =
-            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
-                .unwrap_or_default()
-            {
-                MediaQuality::Balanced => BALANCED_AVATAR_SIZE,
-                MediaQuality::Worse => WORSE_AVATAR_SIZE,
-            };
-
-        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
-        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
-        if let Some(new_name) = self
-            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
-            .await?
-        {
-            self.name = new_name;
+        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
+            return Ok(());
         }
+
+        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
         Ok(())
     }
 
@@ -410,144 +398,40 @@ impl<'a> BlobObject<'a> {
             return Ok(());
         }
 
-        let img_wh =
-            match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?)
-                .unwrap_or_default()
-            {
-                MediaQuality::Balanced => BALANCED_IMAGE_SIZE,
-                MediaQuality::Worse => WORSE_IMAGE_SIZE,
-            };
-
-        if self
-            .recode_to_size(context, blob_abs, img_wh, None)
-            .await?
-            .is_some()
-        {
-            return Err(format_err!(
-                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
-            )
-            .into());
-        }
-        Ok(())
-    }
-
-    async fn recode_to_size(
-        &self,
-        context: &Context,
-        mut blob_abs: PathBuf,
-        mut img_wh: u32,
-        max_bytes: Option<usize>,
-    ) -> Result<Option<String>, BlobError> {
-        let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
-        let orientation = self.get_exif_orientation(context);
-        let mut encoded = Vec::new();
-        let mut changed_name = None;
 
-        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
-            encoded.clear();
-            img.write_to(encoded, image::ImageFormat::Jpeg)?;
-            Ok(())
-        }
-        fn encoded_img_exceeds_bytes(
-            context: &Context,
-            img: &DynamicImage,
-            max_bytes: Option<usize>,
-            encoded: &mut Vec<u8>,
-        ) -> anyhow::Result<bool> {
-            if let Some(max_bytes) = max_bytes {
-                encode_img(img, encoded)?;
-                if encoded.len() > max_bytes {
-                    info!(
-                        context,
-                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
-                        encoded.len(),
-                        img.width(),
-                        img.height(),
-                        max_bytes,
-                    );
-                    return Ok(true);
-                }
-            }
-            Ok(false)
-        }
-        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
 
-        let do_scale =
-            exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
-        let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
-
-        if do_scale || do_rotate {
-            if do_rotate {
-                img = match orientation {
-                    Ok(90) => img.rotate90(),
-                    Ok(180) => img.rotate180(),
-                    Ok(270) => img.rotate270(),
-                    _ => img,
-                }
-            }
-
-            if do_scale {
-                if !exceeds_width {
-                    // The image is already smaller than img_wh, but exceeds max_bytes
-                    // We can directly start with trying to scale down to 2/3 of its current width
-                    img_wh = max(img.width(), img.height()) * 2 / 3
-                }
-
-                loop {
-                    let new_img = img.thumbnail(img_wh, img_wh);
-
-                    if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
-                        if img_wh < 20 {
-                            return Err(format_err!(
-                                "Failed to scale image to below {}B",
-                                max_bytes.unwrap_or_default()
-                            )
-                            .into());
-                        }
-
-                        img_wh = img_wh * 2 / 3;
-                    } else {
-                        if encoded.is_empty() {
-                            encode_img(&new_img, &mut encoded)?;
-                        }
-
-                        info!(
-                            context,
-                            "Final scaled-down image size: {}B ({}px)",
-                            encoded.len(),
-                            img_wh
-                        );
-                        break;
-                    }
-                }
-            }
-
-            // The file format is JPEG now, we may have to change the file extension
-            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
-                blob_abs = blob_abs.with_extension("jpg");
-                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
-                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
-                changed_name = Some(format!("$BLOBDIR/{}", file_name));
-            }
-
-            if encoded.is_empty() {
-                encode_img(&img, &mut encoded)?;
-            }
-
-            fs::write(&blob_abs, &encoded)
-                .await
-                .map_err(|err| BlobError::WriteFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                    cause: err.into(),
-                })?;
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
         }
 
-        Ok(changed_name)
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -611,68 +495,75 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
-    #[error("{0}")]
-    Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
-    use fs::File;
-
     use super::*;
 
-    use crate::{message::Message, test_utils::TestContext};
-    use image::Pixel;
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
-        let fname = t.get_blobdir().join("foo");
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
-        assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
     }
 
     #[async_std::test]
     async fn test_lowercase_ext() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
     #[async_std::test]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
     #[async_std::test]
     async fn test_suffix() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
     #[async_std::test]
     async fn test_create_dup() {
         let t = TestContext::new().await;
-        BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
-        let foo_path = t.get_blobdir().join("foo.txt");
+        BlobObject::create(&t.ctx, "foo.txt", b"hello")
+            .await
+            .unwrap();
+        let foo_path = t.ctx.get_blobdir().join("foo.txt");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
-        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+        BlobObject::create(&t.ctx, "foo.txt", b"world")
+            .await
+            .unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -688,15 +579,15 @@ mod tests {
     #[async_std::test]
     async fn test_double_ext_preserved() {
         let t = TestContext::new().await;
-        BlobObject::create(&t, "foo.tar.gz", b"hello")
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
-        let foo_path = t.get_blobdir().join("foo.tar.gz");
+        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
         assert!(foo_path.exists().await);
-        BlobObject::create(&t, "foo.tar.gz", b"world")
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
             .await
             .unwrap();
-        let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
         while let Some(dirent) = dir.next().await {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
@@ -714,7 +605,7 @@ mod tests {
     async fn test_create_long_names() {
         let t = TestContext::new().await;
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
@@ -724,16 +615,14 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
-            .await
-            .is_err());
-        let whoops = t.get_blobdir().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
 
@@ -743,16 +632,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
-        let src_int = t.get_blobdir().join("internal");
+        let src_int = t.ctx.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -762,9 +649,7 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -819,249 +704,4 @@ mod tests {
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
     }
-
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(&avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-
-        async fn file_size(path_buf: &PathBuf) -> u64 {
-            let file = File::open(path_buf).await.unwrap();
-            file.metadata().await.unwrap().len()
-        }
-
-        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
-
-        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
-            .await
-            .unwrap();
-        assert!(file_size(&avatar_blob).await <= 3000);
-        assert!(file_size(&avatar_blob).await > 2000);
-        let img = image::open(&avatar_blob).unwrap();
-        assert!(img.width() > 130);
-        assert_eq!(img.width(), img.height());
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
-        assert_eq!(
-            avatar_cfg,
-            avatar_src.with_extension("jpg").to_str().unwrap()
-        );
-
-        let img = image::open(avatar_cfg).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[async_std::test]
-    async fn test_recode_image() {
-        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
-        send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
-            .await
-            .unwrap();
-        send_image_check_mediaquality(
-            Some("1"),
-            bytes,
-            1000,
-            1000,
-            0,
-            WORSE_IMAGE_SIZE,
-            WORSE_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-
-        // The "-rotated" files are rotated by 270 degrees using the Exif metadata
-        let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            bytes,
-            2000,
-            1800,
-            270,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let mut bytes = vec![];
-        img_rotated
-            .write_to(&mut bytes, image::ImageFormat::Jpeg)
-            .unwrap();
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let img_rotated = send_image_check_mediaquality(
-            Some("1"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            WORSE_IMAGE_SIZE * 1800 / 2000,
-            WORSE_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
-            .await
-            .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(Some("1"), bytes, 200, 180, 270, 180, 200)
-            .await
-            .unwrap();
-        assert_correct_rotation(&img_rotated);
-    }
-
-    fn assert_correct_rotation(img: &DynamicImage) {
-        // The test images are black in the bottom left corner after correctly applying
-        // the EXIF orientation
-
-        let [luma] = img.get_pixel(10, 10).to_luma().0;
-        assert_eq!(luma, 255);
-        let [luma] = img.get_pixel(img.width() - 10, 10).to_luma().0;
-        assert_eq!(luma, 255);
-        let [luma] = img
-            .get_pixel(img.width() - 10, img.height() - 10)
-            .to_luma()
-            .0;
-        assert_eq!(luma, 255);
-        let [luma] = img.get_pixel(10, img.height() - 10).to_luma().0;
-        assert_eq!(luma, 0);
-    }
-
-    async fn send_image_check_mediaquality(
-        media_quality_config: Option<&str>,
-        bytes: &[u8],
-        original_width: u32,
-        original_height: u32,
-        orientation: i32,
-        compressed_width: u32,
-        compressed_height: u32,
-    ) -> anyhow::Result<DynamicImage> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        alice
-            .set_config(Config::MediaQuality, media_quality_config)
-            .await?;
-        let file = alice.get_blobdir().join("file.jpg");
-
-        File::create(&file).await?.write_all(bytes).await?;
-        let img = image::open(&file)?;
-        assert_eq!(img.width(), original_width);
-        assert_eq!(img.height(), original_height);
-
-        let blob = BlobObject::new_from_path(&alice, &file).await?;
-        assert_eq!(blob.get_exif_orientation(&alice).unwrap_or(0), orientation);
-
-        let mut msg = Message::new(Viewtype::Image);
-        msg.set_file(file.to_str().unwrap(), None);
-        let chat = alice.create_chat(&bob).await;
-        let sent = alice.send_msg(chat.id, &mut msg).await;
-        let alice_msg = alice.get_last_msg().await;
-        assert_eq!(alice_msg.get_width() as u32, compressed_width);
-        assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let img = image::open(alice_msg.get_file(&alice).unwrap())?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
-
-        bob.recv_msg(&sent).await;
-        let bob_msg = bob.get_last_msg().await;
-        assert_eq!(bob_msg.get_width() as u32, compressed_width);
-        assert_eq!(bob_msg.get_height() as u32, compressed_height);
-        let file = bob_msg.get_file(&bob).unwrap();
-
-        let blob = BlobObject::new_from_path(&bob, &file).await?;
-        assert_eq!(blob.get_exif_orientation(&bob).unwrap_or(0), 0);
-
-        let img = image::open(file)?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
-        Ok(img)
-    }
 }

src/chatlist.rs

@@ -1,19 +1,15 @@
-//! # Chat list module.
+//! # Chat list module
 
-use anyhow::{bail, ensure, Result};
-
-use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
-use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
-    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
-    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
-};
-use crate::contact::Contact;
-use crate::context::Context;
+use crate::chat;
+use crate::chat::*;
+use crate::constants::*;
+use crate::contact::*;
+use crate::context::*;
 use crate::ephemeral::delete_expired_messages;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
-use crate::stock_str;
+use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -34,12 +30,15 @@ use crate::stock_str;
 /// and for each messages that is scrolled into view, dc_get_msg() is called then.
 ///
 /// Why no listflags?
-/// Without listflags, dc_get_chatlist() adds the archive "link" automatically as needed.
-/// The UI can just render these items differently then.
+/// Without listflags, dc_get_chatlist() adds the deaddrop and the archive "link" automatically as needed.
+/// The UI can just render these items differently then. Although the deaddrop link is currently always the
+/// first entry and only present on new messages, there is the rough idea that it can be optionally always
+/// present and sorted into the list by date. Rendering the deaddrop in the described way
+/// would not add extra work in the UI then.
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(ChatId, Option<MsgId>)>,
+    ids: Vec<(ChatId, MsgId)>,
 }
 
 impl Chatlist {
@@ -55,6 +54,13 @@ impl Chatlist {
     ///
     /// By default, the function adds some special entries to the list.
     /// These special entries can be identified by the ID returned by chatlist.get_chat_id():
+    /// - DC_CHAT_ID_DEADDROP (1) - this special chat is present if there are
+    ///   messages from addresses that have no relationship to the configured account.
+    ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
+    ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
+    ///   and offers the options "Yes" (call dc_create_chat_by_msg_id()), "Never" (call dc_block_contact())
+    ///   or "Not now".
+    ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -70,9 +76,9 @@ impl Chatlist {
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat and contact requests
+    ///   and hides the device-chat,
     ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
-    /// - if the flag DC_GCL_NO_SPECIALS is set, archive link is not added
+    /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
@@ -102,7 +108,7 @@ impl Chatlist {
 
         let process_row = |row: &rusqlite::Row| {
             let chat_id: ChatId = row.get(0)?;
-            let msg_id: Option<MsgId> = row.get(1)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
             Ok((chat_id, msg_id))
         };
 
@@ -112,9 +118,10 @@ impl Chatlist {
         };
 
         let skip_id = if flag_for_forwarding {
-            ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
-                .await?
+            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .await
                 .unwrap_or_default()
+                .0
         } else {
             ChatId::new(0)
         };
@@ -127,8 +134,13 @@ impl Chatlist {
         //   timestamp
         // - the list starts with the newest chats
         //
-        // The query shows messages from blocked contacts in
-        // groups. Otherwise it would be hard to follow conversations.
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
+        // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
@@ -143,7 +155,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -170,7 +182,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -204,7 +216,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -216,13 +228,14 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let sort_id_up = if flag_for_forwarding {
-                ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
-                    .await?
+                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .await
                     .unwrap_or_default()
+                    .0
             } else {
                 ChatId::new(0)
             };
-            let ids = context.sql.query_map(
+            let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -234,25 +247,34 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND (c.blocked=0 OR (c.blocked=2 AND NOT ?3))
-                   AND NOT c.archived=?4
+                   AND c.blocked=0
+                   AND NOT c.archived=?3
                  GROUP BY c.id
-                 ORDER BY c.id=?5 DESC, c.archived=?6 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsv![MessageState::OutDraft, skip_id, flag_for_forwarding, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             ).await?;
             if !flag_no_specials {
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
+                {
+                    if !flag_for_forwarding {
+                        ids.insert(
+                            0,
+                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                        );
+                    }
+                }
                 add_archived_link_item = true;
             }
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
+                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, None));
+            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -281,7 +303,7 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
         match self.ids.get(index) {
             Some((_chat_id, msg_id)) => Ok(*msg_id),
             None => bail!("Chatlist index out of range"),
@@ -302,19 +324,18 @@ impl Chatlist {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub async fn get_summary(
-        &self,
-        context: &Context,
-        index: usize,
-        chat: Option<&Chat>,
-    ) -> Result<Lot> {
+    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
-            None => bail!("Chatlist index out of range"),
+            None => {
+                let mut ret = Lot::new();
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
+                return Lot::new();
+            }
         };
 
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
@@ -323,50 +344,51 @@ impl Chatlist {
     pub async fn get_summary2(
         context: &Context,
         chat_id: ChatId,
-        lastmsg_id: Option<MsgId>,
+        lastmsg_id: MsgId,
         chat: Option<&Chat>,
-    ) -> Result<Lot> {
+    ) -> Lot {
         let mut ret = Lot::new();
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else {
-            let chat = Chat::load_from_db(context, chat_id).await?;
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
+        } else {
+            return ret;
         };
 
-        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
-            let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == DC_CONTACT_ID_SELF {
-                (Some(lastmsg), None)
-            } else {
-                match chat.typ {
-                    Chattype::Group | Chattype::Mailinglist => {
-                        let lastcontact =
-                            Contact::load_from_db(context, lastmsg.from_id).await.ok();
-                        (Some(lastmsg), lastcontact)
-                    }
-                    Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
-                }
+        let mut lastcontact = None;
+
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
+                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
+            {
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
+
+            Some(lastmsg)
         } else {
-            (None, None)
+            None
         };
 
         if chat.id.is_archived_link() {
             ret.text2 = None;
-        } else if let Some(mut lastmsg) =
-            lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED)
+        } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.fill(&mut lastmsg, chat, lastcontact.as_ref(), context)
-                .await;
+            ret.text2 = Some(
+                context
+                    .stock_str(StockMessage::NoMessages)
+                    .await
+                    .to_string(),
+            );
         } else {
-            ret.text2 = Some(stock_str::no_messages(context).await);
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
+                .await;
         }
 
-        Ok(ret)
+        ret
     }
 
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
@@ -375,42 +397,61 @@ impl Chatlist {
 }
 
 /// Returns the number of archived chats
-pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
-    let count = context
+pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
+    context
         .sql
-        .count(
-            "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
-            paramsv![Blocked::Manually, ChatVisibility::Archived],
+        .query_get_value(
+            context,
+            "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
+            paramsv![],
         )
-        .await?;
-    Ok(count)
+        .await
+        .unwrap_or_default()
+}
+
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context
+        .sql
+        .query_get_value(
+            context,
+            concat!(
+                "SELECT m.id",
+                " FROM msgs m",
+                " LEFT JOIN chats c",
+                "        ON c.id=m.chat_id",
+                " WHERE m.state=10",
+                "   AND m.hidden=0",
+                "   AND c.blocked=2",
+                " ORDER BY m.timestamp DESC, m.id DESC;"
+            ),
+            paramsv![],
+        )
+        .await
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
-    use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
-    use crate::constants::Viewtype;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::stock_str::StockMessage;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_try_load() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
             .await
             .unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -419,24 +460,26 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t, Some(&mut msg)).await.unwrap();
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t, 0, Some("b"), None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
+            .await
+            .unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
         chat_id1
-            .set_visibility(&t, ChatVisibility::Archived)
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
             .await
             .ok();
-        let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
@@ -445,23 +488,23 @@ mod tests {
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
         let t = TestContext::new().await;
-        t.update_device_chats().await.unwrap();
-        create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        t.ctx.update_device_chats().await.unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
 
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
 
-        let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
         assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t, chats.get_chat_id(0))
+        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
             .await
             .unwrap()
             .is_self_talk());
@@ -470,173 +513,49 @@ mod tests {
     #[async_std::test]
     async fn test_search_special_chat_names() {
         let t = TestContext::new().await;
-        t.update_device_chats().await.unwrap();
+        t.ctx.update_device_chats().await.unwrap();
 
-        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 0);
 
-        t.set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+        t.ctx
+            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t, 0, Some("t-1234-s"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
 
-        t.set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+        t.ctx
+            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
             .await
             .unwrap();
-        let chats = Chatlist::try_load(&t, 0, Some("t-5678-b"), None)
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
             .await
             .unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[async_std::test]
-    async fn test_search_single_chat() -> anyhow::Result<()> {
-        let t = TestContext::new_alice().await;
-
-        // receive a one-to-one-message
-        dc_receive_imf(
-            &t,
-            b"From: Bob Authname <bob@example.org>\n\
-                 To: alice@example.com\n\
-                 Subject: foo\n\
-                 Message-ID: <msg1234@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Sun, 22 Mar 2021 22:37:57 +0000\n\
-                 \n\
-                 hello foo\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .await?;
-
-        let chats = Chatlist::try_load(&t, 0, Some("Bob Authname"), None).await?;
-        // Contact request should be searchable
-        assert_eq!(chats.len(), 1);
-
-        let msg = t.get_last_msg().await;
-        let chat_id = msg.get_chat_id();
-        chat_id.accept(&t).await.unwrap();
-
-        let contacts = get_chat_contacts(&t, chat_id).await?;
-        let contact_id = *contacts.first().unwrap();
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Authname");
-
-        // check, the one-to-one-chat can be found using chatlist search query
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // change the name of the contact; this also changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Nickname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 1);
-
-        // revert contact to authname, this again changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Authname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob authname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
-        let t = TestContext::new_alice().await;
-
-        // receive a one-to-one-message without authname set
-        dc_receive_imf(
-            &t,
-            b"From: bob@example.org\n\
-                 To: alice@example.com\n\
-                 Subject: foo\n\
-                 Message-ID: <msg5678@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Sun, 22 Mar 2021 22:38:57 +0000\n\
-                 \n\
-                 hello foo\n",
-            "INBOX",
-            1,
-            false,
-        )
-        .await?;
-
-        let msg = t.get_last_msg().await;
-        let chat_id = msg.get_chat_id();
-        chat_id.accept(&t).await.unwrap();
-        let contacts = get_chat_contacts(&t, chat_id).await?;
-        let contact_id = *contacts.first().unwrap();
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "bob@example.org");
-
-        // check, the one-to-one-chat can be found using chatlist search query
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // change the name of the contact; this also changes the name of the one-to-one-chat
-        let test_id = Contact::create(&t, "Bob Nickname", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "Bob Nickname");
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 0); // email-addresses are searchable in contacts, not in chats
-        let chats = Chatlist::try_load(&t, 0, Some("Bob Nickname"), None).await?;
-        assert_eq!(chats.len(), 1);
-        assert_eq!(chats.get_chat_id(0), chat_id);
-
-        // revert name change, this again changes the name of the one-to-one-chat to the email-address
-        let test_id = Contact::create(&t, "", "bob@example.org").await?;
-        assert_eq!(contact_id, test_id);
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_name(), "bob@example.org");
-        let chats = Chatlist::try_load(&t, 0, Some("bob@example.org"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("bob nickname"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        // finally, also check that a simple substring-search is working with email-addresses
-        let chats = Chatlist::try_load(&t, 0, Some("b@exa"), None).await?;
-        assert_eq!(chats.len(), 1);
-        let chats = Chatlist::try_load(&t, 0, Some("b@exac"), None).await?;
-        assert_eq!(chats.len(), 0);
-
-        Ok(())
-    }
-
     #[async_std::test]
     async fn test_get_summary_unwrap() {
         let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
             .await
             .unwrap();
 
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
+        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
 
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t, 0, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None).await;
         assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -1,62 +0,0 @@
-//! Implementation of Consistent Color Generation.
-//!
-//! Consistent Color Generation is defined in XEP-0392.
-//!
-//! Color Vision Deficiency correction is not implemented as Delta Chat does not offer
-//! corresponding settings.
-use hsluv::hsluv_to_rgb;
-use sha1::{Digest, Sha1};
-
-/// Converts an identifier to Hue angle.
-fn str_to_angle(s: &str) -> f64 {
-    let bytes = s.as_bytes();
-    let result = Sha1::digest(bytes);
-    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
-        + 256 * result.get(1).map_or(0, |&x| u16::from(x));
-    f64::from(checksum) / 65536.0 * 360.0
-}
-
-/// Converts RGB tuple to a 24-bit number.
-///
-/// Returns a 24-bit number with 8 least significant bits corresponding to the blue color and 8
-/// most significant bits corresponding to the red color.
-fn rgb_to_u32((r, g, b): (f64, f64, f64)) -> u32 {
-    let r = ((r * 256.0) as u32).min(255);
-    let g = ((g * 256.0) as u32).min(255);
-    let b = ((b * 256.0) as u32).min(255);
-    65536 * r + 256 * g + b
-}
-
-/// Converts an identifier to RGB color.
-///
-/// Saturation is set to maximum (100.0) to make colors distinguishable, and lightness is set to
-/// half (50.0) to make colors suitable both for light and dark theme.
-pub(crate) fn str_to_color(s: &str) -> u32 {
-    rgb_to_u32(hsluv_to_rgb((str_to_angle(s), 100.0, 50.0)))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_str_to_angle() {
-        // Test against test vectors from
-        // <https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd>
-        assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
-        assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
-        assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);
-        assert!((str_to_angle("council") - 359.994507).abs() < 1e-6);
-        assert!((str_to_angle("Board") - 171.430664).abs() < 1e-6);
-    }
-
-    #[test]
-    fn test_rgb_to_u32() {
-        assert_eq!(rgb_to_u32((0.0, 0.0, 0.0)), 0);
-        assert_eq!(rgb_to_u32((1.0, 1.0, 1.0)), 0xffffff);
-        assert_eq!(rgb_to_u32((0.0, 0.0, 1.0)), 0x0000ff);
-        assert_eq!(rgb_to_u32((0.0, 1.0, 0.0)), 0x00ff00);
-        assert_eq!(rgb_to_u32((1.0, 0.0, 0.0)), 0xff0000);
-        assert_eq!(rgb_to_u32((1.0, 0.5, 0.0)), 0xff8000);
-    }
-}

src/config.rs

@@ -1,6 +1,5 @@
-//! # Key-value configuration management.
+//! # Key-value configuration management
 
-use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -8,28 +7,16 @@ use crate::blob::BlobObject;
 use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
-use crate::dc_tools::{dc_get_abs_path, improve_single_line_input};
+use crate::dc_tools::*;
 use crate::events::EventType;
 use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
-use crate::provider::{get_provider_by_id, Provider};
-use crate::stock_str;
+use crate::stock::StockMessage;
 
 /// The available configuration keys.
 #[derive(
-    Debug,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    Display,
-    EnumString,
-    AsRefStr,
-    EnumIter,
-    EnumProperty,
-    PartialOrd,
-    Ord,
+    Debug, Clone, Copy, PartialEq, Eq, Display, EnumString, AsRefStr, EnumIter, EnumProperty,
 )]
 #[strum(serialize_all = "snake_case")]
 pub enum Config {
@@ -48,11 +35,8 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    Socks5Enabled,
-    Socks5Host,
-    Socks5Port,
-    Socks5User,
-    Socks5Password,
+    #[strum(props(default = "INBOX"))]
+    ImapFolder,
 
     Displayname,
     Selfstatus,
@@ -79,21 +63,12 @@ pub enum Config {
     #[strum(props(default = "1"))]
     MvboxMove,
 
-    #[strum(props(default = "0"))]
-    SentboxMove, // If `MvboxMove` is true, this config is ignored. Currently only used in tests.
-
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
-    /// If set to "1", on the first time `start_io()` is called after configuring,
-    /// the newest existing messages are fetched.
-    /// Existing recipients are added to the contact database regardless of this setting.
-    #[strum(props(default = "1"))]
-    FetchExistingMsgs,
-
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -135,9 +110,6 @@ pub enum Config {
     ConfiguredInboxFolder,
     ConfiguredMvboxFolder,
     ConfiguredSentboxFolder,
-    ConfiguredSpamFolder,
-    ConfiguredTimestamp,
-    ConfiguredProvider,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -149,8 +121,6 @@ pub enum Config {
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
 
-    Bot,
-
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
     #[strum(props(default = "0"))]
@@ -158,142 +128,106 @@ pub enum Config {
 
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
-
-    /// Timestamp of the last time housekeeping was run
-    LastHousekeeping,
-
-    /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
-    #[strum(props(default = "60"))]
-    ScanAllFoldersDebounceSecs,
 }
 
 impl Context {
-    pub async fn config_exists(&self, key: Config) -> Result<bool> {
-        Ok(self.sql.get_raw_config(key).await?.is_some())
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(self, key).await.is_some()
     }
 
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub async fn get_config(&self, key: Config) -> Result<Option<String>> {
+    pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(key).await?;
+                let rel_path = self.sql.get_raw_config(self, key).await;
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(key).await?,
+            _ => self.sql.get_raw_config(self, key).await,
         };
 
         if value.is_some() {
-            return Ok(value);
+            return value;
         }
 
         // Default values
         match key {
-            Config::Selfstatus => Ok(Some(stock_str::status_line(self).await)),
-            Config::ConfiguredInboxFolder => Ok(Some("INBOX".to_owned())),
-            _ => Ok(key.get_str("default").map(|s| s.to_string())),
+            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
+            _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
 
-    pub async fn get_config_int(&self, key: Config) -> Result<i32> {
+    pub async fn get_config_int(&self, key: Config) -> i32 {
         self.get_config(key)
             .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
     }
 
-    pub async fn get_config_i64(&self, key: Config) -> Result<i64> {
-        self.get_config(key)
-            .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
-    }
-
-    pub async fn get_config_u64(&self, key: Config) -> Result<u64> {
-        self.get_config(key)
-            .await
-            .map(|s: Option<String>| s.and_then(|s| s.parse().ok()).unwrap_or_default())
-    }
-
-    pub async fn get_config_bool(&self, key: Config) -> Result<bool> {
-        Ok(self.get_config_int(key).await? != 0)
+    pub async fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key).await != 0
     }
 
     /// Gets configured "delete_server_after" value.
     ///
     /// `None` means never delete the message, `Some(0)` means delete
     /// at once, `Some(x)` means delete after `x` seconds.
-    pub async fn get_config_delete_server_after(&self) -> Result<Option<i64>> {
-        match self.get_config_int(Config::DeleteServerAfter).await? {
-            0 => Ok(None),
-            1 => Ok(Some(0)),
-            x => Ok(Some(x as i64)),
+    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter).await {
+            0 => None,
+            1 => Some(0),
+            x => Some(x as i64),
         }
     }
 
-    /// Gets the configured provider, as saved in the `configured_provider` value.
-    ///
-    /// The provider is determined by `get_provider_info()` during configuration and then saved
-    /// to the db in `param.save_to_database()`, together with all the other `configured_*` values.
-    pub async fn get_configured_provider(&self) -> Result<Option<&'static Provider>> {
-        if let Some(cfg) = self.get_config(Config::ConfiguredProvider).await? {
-            return Ok(get_provider_by_id(&cfg));
-        }
-        Ok(None)
-    }
-
     /// Gets configured "delete_device_after" value.
     ///
     /// `None` means never delete the message, `Some(x)` means delete
     /// after `x` seconds.
-    pub async fn get_config_delete_device_after(&self) -> Result<Option<i64>> {
-        match self.get_config_int(Config::DeleteDeviceAfter).await? {
-            0 => Ok(None),
-            x => Ok(Some(x as i64)),
+    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter).await {
+            0 => None,
+            x => Some(x as i64),
         }
     }
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
                     .execute("UPDATE contacts SET selfavatar_sent=0;", paramsv![])
                     .await?;
                 self.sql
-                    .set_raw_config_bool("attach_selfavatar", true)
+                    .set_raw_config_bool(self, "attach_selfavatar", true)
                     .await?;
                 match value {
                     Some(value) => {
-                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
-                        blob.recode_to_avatar_size(self).await?;
-                        self.sql.set_raw_config(key, Some(blob.as_name())).await?;
-                        Ok(())
-                    }
-                    None => {
-                        self.sql.set_raw_config(key, None).await?;
-                        Ok(())
+                        let blob = BlobObject::new_from_path(&self, value).await?;
+                        blob.recode_to_avatar_size(self)?;
+                        self.sql
+                            .set_raw_config(self, key, Some(blob.as_name()))
+                            .await
                     }
+                    None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
             Config::Selfstatus => {
-                let def = stock_str::status_line(self).await;
+                let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(key, val).await?;
-                Ok(())
+                self.sql.set_raw_config(self, key, val).await
             }
             Config::DeleteDeviceAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
+                let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
                 self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
@@ -303,30 +237,16 @@ impl Context {
             }
             Config::Displayname => {
                 let value = value.map(improve_single_line_input);
-                self.sql.set_raw_config(key, value.as_deref()).await?;
-                Ok(())
+                self.sql.set_raw_config(self, key, value.as_deref()).await
             }
             Config::DeleteServerAfter => {
-                let ret = self
-                    .sql
-                    .set_raw_config(key, value)
-                    .await
-                    .map_err(Into::into);
+                let ret = self.sql.set_raw_config(self, key, value).await;
                 job::schedule_resync(self).await;
                 ret
             }
-            _ => {
-                self.sql.set_raw_config(key, value).await?;
-                Ok(())
-            }
+            _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
-
-    pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { None })
-            .await?;
-        Ok(())
-    }
 }
 
 /// Returns all available configuration keys concated together.
@@ -348,8 +268,12 @@ mod tests {
     use std::string::ToString;
 
     use crate::constants;
-    use crate::test_utils::TestContext;
+    use crate::constants::AVATAR_SIZE;
+    use crate::test_utils::*;
+    use image::GenericImageView;
     use num_traits::FromPrimitive;
+    use std::fs::File;
+    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -363,17 +287,104 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_default_prop() {
+        assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_outside_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(avatar_blob).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_in_blobdir() {
+        let t = TestContext::new().await;
+        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[async_std::test]
+    async fn test_selfavatar_copy_without_recode() {
+        let t = TestContext::new().await;
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
+        assert!(!avatar_blob.exists().await);
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+        assert!(avatar_blob.exists().await);
+        assert_eq!(
+            std::fs::metadata(&avatar_blob).unwrap().len(),
+            avatar_bytes.len() as u64
+        );
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+    }
+
     #[async_std::test]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
         assert_eq!(media_quality, constants::MediaQuality::Balanced);
 
-        t.set_config(Config::MediaQuality, Some("1")).await.unwrap();
+        t.ctx
+            .set_config(Config::MediaQuality, Some("1"))
+            .await
+            .unwrap();
 
-        let media_quality = t.get_config_int(Config::MediaQuality).await.unwrap();
+        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 1);
         assert_eq!(constants::MediaQuality::Worse as i32, 1);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure.rs

@@ -1,694 +0,0 @@
-//! Email accounts autoconfiguration process module.
-
-mod auto_mozilla;
-mod auto_outlook;
-mod read_url;
-mod server_params;
-
-use anyhow::{bail, ensure, Context as _, Result};
-use async_std::prelude::*;
-use async_std::task;
-use itertools::Itertools;
-use job::Action;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
-
-use crate::dc_tools::EmailAddress;
-use crate::imap::Imap;
-use crate::login_param::Socks5Config;
-use crate::login_param::{LoginParam, ServerLoginParam};
-use crate::message::Message;
-use crate::oauth2::dc_get_oauth2_addr;
-use crate::provider::{Protocol, Socket, UsernamePattern};
-use crate::smtp::Smtp;
-use crate::stock_str;
-use crate::{chat, e2ee, provider};
-use crate::{config::Config, dc_tools::time};
-use crate::{
-    constants::{Viewtype, DC_LP_AUTH_FLAGS, DC_LP_AUTH_NORMAL, DC_LP_AUTH_OAUTH2},
-    job,
-};
-use crate::{context::Context, param::Params};
-
-use auto_mozilla::moz_autoconfigure;
-use auto_outlook::outlk_autodiscover;
-use server_params::{expand_param_vector, ServerParams};
-
-macro_rules! progress {
-    ($context:tt, $progress:expr, $comment:expr) => {
-        assert!(
-            $progress <= 1000,
-            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
-        );
-        $context.emit_event($crate::events::EventType::ConfigureProgress {
-            progress: $progress,
-            comment: $comment,
-        });
-    };
-    ($context:tt, $progress:expr) => {
-        progress!($context, $progress, None);
-    };
-}
-
-impl Context {
-    /// Checks if the context is already configured.
-    pub async fn is_configured(&self) -> Result<bool> {
-        self.sql
-            .get_raw_config_bool("configured")
-            .await
-            .map_err(Into::into)
-    }
-
-    /// Configures this account with the currently set parameters.
-    pub async fn configure(&self) -> Result<()> {
-        use futures::future::FutureExt;
-
-        ensure!(
-            !self.scheduler.read().await.is_running(),
-            "cannot configure, already running"
-        );
-        ensure!(
-            self.sql.is_open().await,
-            "cannot configure, database not opened."
-        );
-        let cancel_channel = self.alloc_ongoing().await?;
-
-        let res = self
-            .inner_configure()
-            .race(cancel_channel.recv().map(|_| {
-                progress!(self, 0);
-                Ok(())
-            }))
-            .await;
-
-        self.free_ongoing().await;
-
-        res
-    }
-
-    async fn inner_configure(&self) -> Result<()> {
-        info!(self, "Configure ...");
-
-        let mut param = LoginParam::from_database(self, "").await?;
-        let success = configure(self, &mut param).await;
-        self.set_config(Config::NotifyAboutWrongPw, None).await?;
-
-        if let Some(provider) = param.provider {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    if !self.config_exists(def.key).await? {
-                        info!(self, "apply config_defaults {}={}", def.key, def.value);
-                        self.set_config(def.key, Some(def.value)).await?;
-                    } else {
-                        info!(
-                            self,
-                            "skip already set config_defaults {}={}", def.key, def.value
-                        );
-                    }
-                }
-            }
-
-            if !provider.after_login_hint.is_empty() {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(provider.after_login_hint.to_string());
-                if chat::add_device_msg(self, Some("core-provider-info"), Some(&mut msg))
-                    .await
-                    .is_err()
-                {
-                    warn!(self, "cannot add after_login_hint as core-provider-info");
-                }
-            }
-        }
-
-        match success {
-            Ok(_) => {
-                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
-                    .await?;
-                progress!(self, 1000);
-                Ok(())
-            }
-            Err(err) => {
-                progress!(
-                    self,
-                    0,
-                    Some(
-                        stock_str::configuration_failed(
-                            self,
-                            // We are using Anyhow's .context() and to show the
-                            // inner error, too, we need the {:#}:
-                            format!("{:#}", err),
-                        )
-                        .await
-                    )
-                );
-                Err(err)
-            }
-        }
-    }
-}
-
-async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
-    progress!(ctx, 1);
-
-    // Check basic settings.
-    ensure!(!param.addr.is_empty(), "Please enter an email address.");
-
-    // Only check for IMAP password, SMTP password is an "advanced" setting.
-    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
-    if param.smtp.password.is_empty() {
-        param.smtp.password = param.imap.password.clone()
-    }
-
-    // Normalize authentication flags.
-    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
-        DC_LP_AUTH_OAUTH2 => true,
-        DC_LP_AUTH_NORMAL => false,
-        _ => false,
-    };
-    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
-    param.server_flags |= if oauth2 {
-        DC_LP_AUTH_OAUTH2 as i32
-    } else {
-        DC_LP_AUTH_NORMAL as i32
-    };
-
-    let socks5_config = param.socks5_config.clone();
-    let socks5_enabled = socks5_config.is_some();
-
-    let ctx2 = ctx.clone();
-    let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
-
-    // Step 1: Load the parameters and check email-address and password
-
-    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too
-    if oauth2 && !socks5_enabled {
-        // the used oauth2 addr may differ, check this.
-        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
-        progress!(ctx, 10);
-        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
-            .await?
-            .and_then(|e| e.parse().ok())
-        {
-            info!(ctx, "Authorized address is {}", oauth2_addr);
-            param.addr = oauth2_addr;
-            ctx.sql
-                .set_raw_config("addr", Some(param.addr.as_str()))
-                .await?;
-        }
-        progress!(ctx, 20);
-    }
-    // no oauth? - just continue it's no error
-
-    let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
-    let param_domain = parsed.domain;
-    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
-
-    // Step 2: Autoconfig
-    progress!(ctx, 200);
-
-    let param_autoconfig;
-    if param.imap.server.is_empty()
-        && param.imap.port == 0
-        && param.imap.security == Socket::Automatic
-        && param.imap.user.is_empty()
-        && param.smtp.server.is_empty()
-        && param.smtp.port == 0
-        && param.smtp.security == Socket::Automatic
-        && param.smtp.user.is_empty()
-    {
-        // no advanced parameters entered by the user: query provider-database or do Autoconfig
-
-        info!(
-            ctx,
-            "checking internal provider-info for offline autoconfig"
-        );
-
-        if let Some(provider) = provider::get_provider_info(&param_domain, socks5_enabled).await {
-            param.provider = Some(provider);
-            match provider.status {
-                provider::Status::Ok | provider::Status::Preparation => {
-                    if provider.server.is_empty() {
-                        info!(ctx, "offline autoconfig found, but no servers defined");
-                        param_autoconfig = None;
-                    } else {
-                        info!(ctx, "offline autoconfig found");
-                        let servers = provider
-                            .server
-                            .iter()
-                            .map(|s| ServerParams {
-                                protocol: s.protocol,
-                                socket: s.socket,
-                                hostname: s.hostname.to_string(),
-                                port: s.port,
-                                username: match s.username_pattern {
-                                    UsernamePattern::Email => param.addr.to_string(),
-                                    UsernamePattern::Emaillocalpart => {
-                                        if let Some(at) = param.addr.find('@') {
-                                            param.addr.split_at(at).0.to_string()
-                                        } else {
-                                            param.addr.to_string()
-                                        }
-                                    }
-                                },
-                            })
-                            .collect();
-
-                        param_autoconfig = Some(servers)
-                    }
-                }
-                provider::Status::Broken => {
-                    info!(ctx, "offline autoconfig found, provider is broken");
-                    param_autoconfig = None;
-                }
-            }
-        } else {
-            // Try receiving autoconfig
-            info!(ctx, "no offline autoconfig found");
-            param_autoconfig = if socks5_enabled {
-                // Currently we can't do http requests through socks5, to not leak
-                // the ip, just don't do online autoconfig
-                info!(ctx, "socks5 enabled, skipping autoconfig");
-                None
-            } else {
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
-            }
-        }
-    } else {
-        param_autoconfig = None;
-    }
-
-    progress!(ctx, 500);
-
-    let mut servers = param_autoconfig.unwrap_or_default();
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::Imap)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::Imap,
-            hostname: param.imap.server.clone(),
-            port: param.imap.port,
-            socket: param.imap.security,
-            username: param.imap.user.clone(),
-        })
-    }
-    if !servers
-        .iter()
-        .any(|server| server.protocol == Protocol::Smtp)
-    {
-        servers.push(ServerParams {
-            protocol: Protocol::Smtp,
-            hostname: param.smtp.server.clone(),
-            port: param.smtp.port,
-            socket: param.smtp.security,
-            username: param.smtp.user.clone(),
-        })
-    }
-    let servers = expand_param_vector(servers, &param.addr, &param_domain);
-
-    progress!(ctx, 550);
-
-    // Spawn SMTP configuration task
-    let mut smtp = Smtp::new();
-
-    let context_smtp = ctx.clone();
-    let mut smtp_param = param.smtp.clone();
-    let smtp_addr = param.addr.clone();
-    let smtp_servers: Vec<ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Smtp)
-        .cloned()
-        .collect();
-    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
-
-    let smtp_config_task = task::spawn(async move {
-        let mut smtp_configured = false;
-        let mut errors = Vec::new();
-        for smtp_server in smtp_servers {
-            smtp_param.user = smtp_server.username.clone();
-            smtp_param.server = smtp_server.hostname.clone();
-            smtp_param.port = smtp_server.port;
-            smtp_param.security = smtp_server.socket;
-
-            match try_smtp_one_param(
-                &context_smtp,
-                &smtp_param,
-                &socks5_config,
-                &smtp_addr,
-                oauth2,
-                provider_strict_tls,
-                &mut smtp,
-            )
-            .await
-            {
-                Ok(_) => {
-                    smtp_configured = true;
-                    break;
-                }
-                Err(e) => errors.push(e),
-            }
-        }
-
-        if smtp_configured {
-            Ok(smtp_param)
-        } else {
-            Err(errors)
-        }
-    });
-
-    progress!(ctx, 600);
-
-    // Configure IMAP
-
-    let mut imap: Option<Imap> = None;
-    let imap_servers: Vec<&ServerParams> = servers
-        .iter()
-        .filter(|params| params.protocol == Protocol::Imap)
-        .collect();
-    let imap_servers_count = imap_servers.len();
-    let mut errors = Vec::new();
-    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
-        param.imap.user = imap_server.username.clone();
-        param.imap.server = imap_server.hostname.clone();
-        param.imap.port = imap_server.port;
-        param.imap.security = imap_server.socket;
-
-        match try_imap_one_param(
-            ctx,
-            &param.imap,
-            &param.socks5_config,
-            &param.addr,
-            oauth2,
-            provider_strict_tls,
-        )
-        .await
-        {
-            Ok(configured_imap) => {
-                imap = Some(configured_imap);
-                break;
-            }
-            Err(e) => errors.push(e),
-        }
-        progress!(
-            ctx,
-            600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
-        );
-    }
-    let mut imap = match imap {
-        Some(imap) => imap,
-        None => bail!(nicer_configuration_error(ctx, errors).await),
-    };
-
-    progress!(ctx, 850);
-
-    // Wait for SMTP configuration
-    match smtp_config_task.await {
-        Ok(smtp_param) => {
-            param.smtp = smtp_param;
-        }
-        Err(errors) => {
-            bail!(nicer_configuration_error(ctx, errors).await);
-        }
-    }
-
-    progress!(ctx, 900);
-
-    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await?
-        || ctx.get_config_bool(Config::MvboxMove).await?;
-
-    imap.configure_folders(ctx, create_mvbox).await?;
-
-    imap.select_with_uidvalidity(ctx, "INBOX")
-        .await
-        .context("could not read INBOX status")?;
-
-    drop(imap);
-
-    progress!(ctx, 910);
-    // configuration success - write back the configured parameters with the
-    // "configured_" prefix; also write the "configured"-flag */
-    // the trailing underscore is correct
-    param.save_to_database(ctx, "configured_").await?;
-    ctx.sql.set_raw_config_bool("configured", true).await?;
-    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
-        .await?;
-
-    progress!(ctx, 920);
-
-    e2ee::ensure_secret_key_exists(ctx).await?;
-    info!(ctx, "key generation completed");
-
-    job::add(
-        ctx,
-        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
-    )
-    .await;
-
-    progress!(ctx, 940);
-    update_device_chats_handle.await?;
-
-    Ok(())
-}
-
-/// Retrieve available autoconfigurations.
-///
-/// A Search configurations from the domain used in the email-address, prefer encrypted
-/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
-async fn get_autoconfig(
-    ctx: &Context,
-    param: &LoginParam,
-    param_domain: &str,
-    param_addr_urlencoded: &str,
-) -> Option<Vec<ServerParams>> {
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        &format!(
-            "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
-            param_domain, param_addr_urlencoded
-        ),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 300);
-
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see <https://releases.mozilla.org/pub/thunderbird/>,  which makes some sense
-        &format!(
-            "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
-            &param_domain, &param_addr_urlencoded
-        ),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 310);
-
-    // Outlook uses always SSL but different domains (this comment describes the next two steps)
-    if let Ok(res) = outlk_autodiscover(
-        ctx,
-        format!("https://{}/autodiscover/autodiscover.xml", &param_domain),
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 320);
-
-    if let Ok(res) = outlk_autodiscover(
-        ctx,
-        format!(
-            "https://autodiscover.{}/autodiscover/autodiscover.xml",
-            &param_domain
-        ),
-    )
-    .await
-    {
-        return Some(res);
-    }
-    progress!(ctx, 330);
-
-    // always SSL for Thunderbird's database
-    if let Ok(res) = moz_autoconfigure(
-        ctx,
-        &format!("https://autoconfig.thunderbird.net/v1.1/{}", &param_domain),
-        param,
-    )
-    .await
-    {
-        return Some(res);
-    }
-
-    None
-}
-
-async fn try_imap_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
-    addr: &str,
-    oauth2: bool,
-    provider_strict_tls: bool,
-) -> Result<Imap, ConfigurationError> {
-    let inf = format!(
-        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
-        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
-    );
-    info!(context, "Trying: {}", inf);
-
-    let (_s, r) = async_std::channel::bounded(1);
-
-    let mut imap = match Imap::new(
-        param,
-        socks5_config.clone(),
-        addr,
-        oauth2,
-        provider_strict_tls,
-        r,
-    )
-    .await
-    {
-        Err(err) => {
-            info!(context, "failure: {}", err);
-            return Err(ConfigurationError {
-                config: inf,
-                msg: err.to_string(),
-            });
-        }
-        Ok(imap) => imap,
-    };
-
-    match imap.connect(context).await {
-        Err(err) => {
-            info!(context, "failure: {}", err);
-            Err(ConfigurationError {
-                config: inf,
-                msg: err.to_string(),
-            })
-        }
-        Ok(()) => {
-            info!(context, "success: {}", inf);
-            Ok(imap)
-        }
-    }
-}
-
-async fn try_smtp_one_param(
-    context: &Context,
-    param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
-    addr: &str,
-    oauth2: bool,
-    provider_strict_tls: bool,
-    smtp: &mut Smtp,
-) -> Result<(), ConfigurationError> {
-    let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
-        param.user,
-        param.server,
-        param.port,
-        param.security,
-        param.certificate_checks,
-        oauth2,
-        if let Some(socks5_config) = socks5_config {
-            socks5_config.to_string()
-        } else {
-            "None".to_string()
-        }
-    );
-    info!(context, "Trying: {}", inf);
-
-    if let Err(err) = smtp
-        .connect(
-            context,
-            param,
-            socks5_config,
-            addr,
-            oauth2,
-            provider_strict_tls,
-        )
-        .await
-    {
-        info!(context, "failure: {}", err);
-        Err(ConfigurationError {
-            config: inf,
-            msg: err.to_string(),
-        })
-    } else {
-        info!(context, "success: {}", inf);
-        smtp.disconnect().await;
-        Ok(())
-    }
-}
-
-#[derive(Debug, thiserror::Error)]
-#[error("Trying {config}…\nError: {msg}")]
-pub struct ConfigurationError {
-    config: String,
-    msg: String,
-}
-
-async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
-    let first_err = if let Some(f) = errors.first() {
-        f
-    } else {
-        // This means configuration failed but no errors have been captured. This should never
-        // happen, but if it does, the user will see classic "Error: no error".
-        return "no error".to_string();
-    };
-
-    if errors
-        .iter()
-        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
-    {
-        return stock_str::error_no_network(context).await;
-    }
-
-    if errors.iter().all(|e| e.msg == first_err.msg) {
-        return first_err.msg.to_string();
-    }
-
-    errors.iter().map(|e| e.to_string()).join("\n\n")
-}
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("Invalid email address: {0:?}")]
-    InvalidEmailAddress(String),
-
-    #[error("XML error at position {position}: {error}")]
-    InvalidXml {
-        position: usize,
-        #[source]
-        error: quick_xml::Error,
-    },
-
-    #[error("Failed to get URL: {0}")]
-    ReadUrl(#[from] self::read_url::Error),
-
-    #[error("Number of redirection is exceeded")]
-    Redirection,
-}
-
-#[cfg(test)]
-mod tests {
-    #![allow(clippy::indexing_slicing)]
-
-    use crate::config::Config;
-    use crate::test_utils::TestContext;
-
-    #[async_std::test]
-    async fn test_no_panic_on_bad_credentials() {
-        let t = TestContext::new().await;
-        t.set_config(Config::Addr, Some("probably@unexistant.addr"))
-            .await
-            .unwrap();
-        t.set_config(Config::MailPw, Some("123456")).await.unwrap();
-        assert!(t.configure().await.is_err());
-    }
-}

src/configure/auto_mozilla.rs

@@ -1,6 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: <https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
 use quick_xml::events::{BytesStart, Event};
 
 use std::io::BufRead;
@@ -115,8 +115,8 @@ fn parse_server<B: BufRead>(
                     MozConfigTag::Username => username = Some(val),
                     MozConfigTag::Sockettype => {
                         sockettype = match val.to_lowercase().as_ref() {
-                            "ssl" => Socket::Ssl,
-                            "starttls" => Socket::Starttls,
+                            "ssl" => Socket::SSL,
+                            "starttls" => Socket::STARTTLS,
                             "plain" => Socket::Plain,
                             _ => Socket::Automatic,
                         }
@@ -233,8 +233,8 @@ fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerPar
         .chain(moz_ac.outgoing_servers.into_iter())
         .filter_map(|server| {
             let protocol = match server.typ.as_ref() {
-                "imap" => Some(Protocol::Imap),
-                "smtp" => Some(Protocol::Smtp),
+                "imap" => Some(Protocol::IMAP),
+                "smtp" => Some(Protocol::SMTP),
                 _ => None,
             };
             Some(ServerParams {
@@ -276,10 +276,10 @@ mod tests {
     fn test_parse_outlook_autoconfig() {
         let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
         let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res[0].protocol, Protocol::Imap);
+        assert_eq!(res[0].protocol, Protocol::IMAP);
         assert_eq!(res[0].hostname, "outlook.office365.com");
         assert_eq!(res[0].port, 993);
-        assert_eq!(res[1].protocol, Protocol::Smtp);
+        assert_eq!(res[1].protocol, Protocol::SMTP);
         assert_eq!(res[1].hostname, "smtp.office365.com");
         assert_eq!(res[1].port, 587);
     }
@@ -295,25 +295,25 @@ mod tests {
         assert_eq!(res.incoming_servers[0].typ, "imap");
         assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[0].port, 993);
-        assert_eq!(res.incoming_servers[0].sockettype, Socket::Ssl);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
         assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[1].typ, "imap");
         assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[1].port, 143);
-        assert_eq!(res.incoming_servers[1].sockettype, Socket::Starttls);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
         assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[2].typ, "pop3");
         assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[2].port, 995);
-        assert_eq!(res.incoming_servers[2].sockettype, Socket::Ssl);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
         assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
 
         assert_eq!(res.incoming_servers[3].typ, "pop3");
         assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
         assert_eq!(res.incoming_servers[3].port, 110);
-        assert_eq!(res.incoming_servers[3].sockettype, Socket::Starttls);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
         assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
 
         assert_eq!(res.outgoing_servers.len(), 1);
@@ -321,7 +321,7 @@ mod tests {
         assert_eq!(res.outgoing_servers[0].typ, "smtp");
         assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
         assert_eq!(res.outgoing_servers[0].port, 587);
-        assert_eq!(res.outgoing_servers[0].sockettype, Socket::Starttls);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
         assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -15,27 +15,27 @@ use super::{Error, ServerParams};
 
 /// Result of parsing a single `Protocol` tag.
 ///
-/// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox>
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
 #[derive(Debug)]
 struct ProtocolTag {
     /// Server type, such as "IMAP", "SMTP" or "POP3".
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
     pub typ: String,
 
     /// Server identifier, hostname or IP address for IMAP and SMTP.
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
     pub server: String,
 
     /// Network port.
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
     pub port: u16,
 
     /// Whether connection should be secure, "on" or "off", default is "on".
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
     pub ssl: bool,
 }
 
@@ -73,7 +73,7 @@ fn parse_protocol<B: BufRead>(
                 }
             }
             Event::Text(ref e) => {
-                let val = e.unescape_and_decode(reader).unwrap_or_default();
+                let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
@@ -117,7 +117,7 @@ fn parse_redirecturl<B: BufRead>(
     let mut buf = Vec::new();
     match reader.read_event(&mut buf)? {
         Event::Text(ref e) => {
-            let val = e.unescape_and_decode(reader).unwrap_or_default();
+            let val = e.unescape_and_decode(&reader).unwrap_or_default();
             Ok(val.trim().to_string())
         }
         _ => Ok("".to_string()),
@@ -154,7 +154,7 @@ fn parse_xml_reader<B: BufRead>(
 }
 
 fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut reader = quick_xml::Reader::from_str(xml_raw);
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
     reader.trim_text(true);
 
     parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
@@ -169,8 +169,8 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
         .filter_map(|protocol| {
             Some(ServerParams {
                 protocol: match protocol.typ.to_lowercase().as_ref() {
-                    "imap" => Some(Protocol::Imap),
-                    "smtp" => Some(Protocol::Smtp),
+                    "imap" => Some(Protocol::IMAP),
+                    "smtp" => Some(Protocol::SMTP),
                     _ => None,
                 }?,
                 socket: match protocol.ssl {
@@ -187,8 +187,9 @@ fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
 
 pub(crate) async fn outlk_autodiscover(
     context: &Context,
-    mut url: String,
+    url: &str,
 ) -> Result<Vec<ServerParams>, Error> {
+    let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
         let xml_raw = read_url(context, &url).await?;
@@ -203,7 +204,7 @@ pub(crate) async fn outlk_autodiscover(
             }
         }
     }
-    Err(Error::Redirection)
+    Err(Error::RedirectionError)
 }
 
 #[cfg(test)]

src/configure/mod.rs

@@ -0,0 +1,629 @@
+//! Email accounts autoconfiguration process module
+
+mod auto_mozilla;
+mod auto_outlook;
+mod read_url;
+mod server_params;
+
+use crate::config::Config;
+use crate::constants::*;
+use crate::context::Context;
+use crate::dc_tools::*;
+use crate::imap::Imap;
+use crate::job;
+use crate::login_param::{LoginParam, ServerLoginParam};
+use crate::message::Message;
+use crate::oauth2::*;
+use crate::param::Params;
+use crate::provider::{Protocol, Socket, UsernamePattern};
+use crate::smtp::Smtp;
+use crate::stock::StockMessage;
+use crate::EventType;
+use crate::{chat, e2ee, provider};
+use anyhow::{bail, ensure, Context as _, Result};
+use async_std::prelude::*;
+use async_std::task;
+use auto_mozilla::moz_autoconfigure;
+use auto_outlook::outlk_autodiscover;
+use itertools::Itertools;
+use job::Action;
+use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use server_params::{expand_param_vector, ServerParams};
+
+impl Context {
+    /// Checks if the context is already configured.
+    pub async fn is_configured(&self) -> bool {
+        self.sql.get_raw_config_bool(self, "configured").await
+    }
+
+    /// Configures this account with the currently set parameters.
+    pub async fn configure(&self) -> Result<()> {
+        use futures::future::FutureExt;
+
+        ensure!(
+            !self.scheduler.read().await.is_running(),
+            "cannot configure, already running"
+        );
+        ensure!(
+            self.sql.is_open().await,
+            "cannot configure, database not opened."
+        );
+        let cancel_channel = self.alloc_ongoing().await?;
+
+        let ctx2 = self.clone();
+        let progress = ProgressHandler::new(15.0, move |p| {
+            ctx2.emit_event(EventType::ConfigureProgress {
+                progress: p,
+                comment: None,
+            });
+        });
+
+        let res = self
+            .inner_configure(&progress)
+            .race(cancel_channel.recv().map(|_| {
+                progress.p(0);
+                Ok(())
+            }))
+            .await;
+
+        self.free_ongoing().await;
+
+        res
+    }
+
+    async fn inner_configure(&self, progress: &impl Progress) -> Result<()> {
+        info!(self, "Configure ...");
+
+        let mut param = LoginParam::from_database(self, "").await;
+        let success = configure(self, &mut param, progress).await;
+        self.set_config(Config::NotifyAboutWrongPw, None).await?;
+
+        if let Some(provider) = provider::get_provider_info(&param.addr) {
+            if let Some(config_defaults) = &provider.config_defaults {
+                for def in config_defaults.iter() {
+                    if !self.config_exists(def.key).await {
+                        info!(self, "apply config_defaults {}={}", def.key, def.value);
+                        self.set_config(def.key, Some(def.value)).await?;
+                    } else {
+                        info!(
+                            self,
+                            "skip already set config_defaults {}={}", def.key, def.value
+                        );
+                    }
+                }
+            }
+
+            if !provider.after_login_hint.is_empty() {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some(provider.after_login_hint.to_string());
+                if chat::add_device_msg(self, Some("core-provider-info"), Some(&mut msg))
+                    .await
+                    .is_err()
+                {
+                    warn!(self, "cannot add after_login_hint as core-provider-info");
+                }
+            }
+        }
+
+        match success {
+            Ok(_) => {
+                self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+                    .await?;
+                progress.p(1000);
+                Ok(())
+            }
+            Err(err) => {
+                progress.kill().await;
+                emit_event!(
+                    self,
+                    EventType::ConfigureProgress {
+                        progress: 0,
+                        comment: Some(
+                            self.stock_string_repl_str(
+                                StockMessage::ConfigurationFailed,
+                                // We are using Anyhow's .context() and to show the inner error too, we need the {:#}:
+                                format!("{:#}", err),
+                            )
+                            .await
+                        )
+                    }
+                );
+                Err(err)
+            }
+        }
+    }
+}
+
+async fn configure(ctx: &Context, param: &mut LoginParam, progress: &impl Progress) -> Result<()> {
+    progress.p(1);
+
+    // Check basic settings.
+    ensure!(!param.addr.is_empty(), "Please enter an email address.");
+
+    // Only check for IMAP password, SMTP password is an "advanced" setting.
+    ensure!(!param.imap.password.is_empty(), "Please enter a password.");
+    if param.smtp.password.is_empty() {
+        param.smtp.password = param.imap.password.clone()
+    }
+
+    // Normalize authentication flags.
+    let oauth2 = match param.server_flags & DC_LP_AUTH_FLAGS as i32 {
+        DC_LP_AUTH_OAUTH2 => true,
+        DC_LP_AUTH_NORMAL => false,
+        _ => false,
+    };
+    param.server_flags &= !(DC_LP_AUTH_FLAGS as i32);
+    param.server_flags |= if oauth2 {
+        DC_LP_AUTH_OAUTH2 as i32
+    } else {
+        DC_LP_AUTH_NORMAL as i32
+    };
+
+    // Step 1: Load the parameters and check email-address and password
+
+    if oauth2 {
+        // the used oauth2 addr may differ, check this.
+        // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
+        progress.p(10);
+        if let Some(oauth2_addr) = dc_get_oauth2_addr(ctx, &param.addr, &param.imap.password)
+            .await
+            .and_then(|e| e.parse().ok())
+        {
+            info!(ctx, "Authorized address is {}", oauth2_addr);
+            param.addr = oauth2_addr;
+            ctx.sql
+                .set_raw_config(ctx, "addr", Some(param.addr.as_str()))
+                .await?;
+        }
+        progress.p(20);
+    }
+    // no oauth? - just continue it's no error
+
+    let parsed: EmailAddress = param.addr.parse().context("Bad email-address")?;
+    let param_domain = parsed.domain;
+    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
+
+    // Step 2: Autoconfig
+    progress.p(200);
+
+    let param_autoconfig;
+    if param.imap.server.is_empty()
+        && param.imap.port == 0
+        && param.imap.security == Socket::Automatic
+        && param.imap.user.is_empty()
+        && param.smtp.server.is_empty()
+        && param.smtp.port == 0
+        && param.smtp.security == Socket::Automatic
+        && param.smtp.user.is_empty()
+    {
+        // no advanced parameters entered by the user: query provider-database or do Autoconfig
+
+        if let Some(servers) = get_offline_autoconfig(ctx, &param.addr) {
+            param_autoconfig = Some(servers);
+        } else {
+            param_autoconfig =
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded, progress).await;
+        }
+    } else {
+        param_autoconfig = None;
+    }
+
+    progress.p(500);
+
+    let servers = expand_param_vector(
+        param_autoconfig.unwrap_or_else(|| {
+            vec![
+                ServerParams {
+                    protocol: Protocol::IMAP,
+                    hostname: param.imap.server.clone(),
+                    port: param.imap.port,
+                    socket: param.imap.security,
+                    username: param.imap.user.clone(),
+                },
+                ServerParams {
+                    protocol: Protocol::SMTP,
+                    hostname: param.smtp.server.clone(),
+                    port: param.smtp.port,
+                    socket: param.smtp.security,
+                    username: param.smtp.user.clone(),
+                },
+            ]
+        }),
+        &param.addr,
+        &param_domain,
+    );
+
+    progress.p(550);
+
+    // Spawn SMTP configuration task
+    let mut smtp = Smtp::new();
+
+    let context_smtp = ctx.clone();
+    let mut smtp_param = param.smtp.clone();
+    let smtp_addr = param.addr.clone();
+    let smtp_servers: Vec<ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::SMTP)
+        .cloned()
+        .collect();
+
+    let smtp_config_task = task::spawn(async move {
+        let mut smtp_configured = false;
+        let mut errors = Vec::new();
+        for smtp_server in smtp_servers {
+            smtp_param.user = smtp_server.username.clone();
+            smtp_param.server = smtp_server.hostname.clone();
+            smtp_param.port = smtp_server.port;
+            smtp_param.security = smtp_server.socket;
+
+            match try_smtp_one_param(&context_smtp, &smtp_param, &smtp_addr, oauth2, &mut smtp)
+                .await
+            {
+                Ok(_) => {
+                    smtp_configured = true;
+                    break;
+                }
+                Err(e) => errors.push(e),
+            }
+        }
+
+        if smtp_configured {
+            Ok(smtp_param)
+        } else {
+            Err(errors)
+        }
+    });
+
+    progress.p(600);
+
+    // Configure IMAP
+    let (_s, r) = async_std::sync::channel(1);
+    let mut imap = Imap::new(r);
+
+    let mut imap_configured = false;
+    let imap_servers: Vec<&ServerParams> = servers
+        .iter()
+        .filter(|params| params.protocol == Protocol::IMAP)
+        .collect();
+    let imap_servers_count = imap_servers.len();
+    let mut errors = Vec::new();
+    for (imap_server_index, imap_server) in imap_servers.into_iter().enumerate() {
+        param.imap.user = imap_server.username.clone();
+        param.imap.server = imap_server.hostname.clone();
+        param.imap.port = imap_server.port;
+        param.imap.security = imap_server.socket;
+
+        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, &mut imap).await {
+            Ok(_) => {
+                imap_configured = true;
+                break;
+            }
+            Err(e) => errors.push(e),
+        }
+        progress.p(600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count);
+    }
+    if !imap_configured {
+        bail!(nicer_configuration_error(ctx, errors).await);
+    }
+
+    progress.p(850);
+
+    // Wait for SMTP configuration
+    match smtp_config_task.await {
+        Ok(smtp_param) => {
+            param.smtp = smtp_param;
+        }
+        Err(errors) => {
+            bail!(nicer_configuration_error(ctx, errors).await);
+        }
+    }
+
+    progress.p(900);
+
+    let create_mvbox = ctx.get_config_bool(Config::MvboxWatch).await
+        || ctx.get_config_bool(Config::MvboxMove).await;
+
+    imap.configure_folders(ctx, create_mvbox).await?;
+
+    imap.select_with_uidvalidity(ctx, "INBOX")
+        .await
+        .context("could not read INBOX status")?;
+
+    drop(imap);
+
+    progress.p(910);
+    // configuration success - write back the configured parameters with the
+    // "configured_" prefix; also write the "configured"-flag */
+    // the trailing underscore is correct
+    param.save_to_database(ctx, "configured_").await?;
+    ctx.sql.set_raw_config_bool(ctx, "configured", true).await?;
+
+    progress.p(920);
+
+    e2ee::ensure_secret_key_exists(ctx).await?;
+    info!(ctx, "key generation completed");
+
+    job::add(
+        ctx,
+        job::Job::new(Action::FetchExistingMsgs, 0, Params::new(), 0),
+    )
+    .await;
+
+    progress.p(940);
+
+    Ok(())
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum AutoconfigProvider {
+    Mozilla,
+    Outlook,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct AutoconfigSource {
+    provider: AutoconfigProvider,
+    url: String,
+}
+
+impl AutoconfigSource {
+    fn all(domain: &str, addr: &str) -> [Self; 5] {
+        [
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
+                    "https://autoconfig.{}/mail/config-v1.1.xml?emailaddress={}",
+                    domain, addr,
+                ),
+            },
+            // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!(
+                    "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
+                    domain, addr
+                ),
+            },
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!("https://{}/autodiscover/autodiscover.xml", domain),
+            },
+            // Outlook uses always SSL but different domains (this comment describes the next two steps)
+            AutoconfigSource {
+                provider: AutoconfigProvider::Outlook,
+                url: format!(
+                    "https://autodiscover.{}/autodiscover/autodiscover.xml",
+                    domain
+                ),
+            },
+            // always SSL for Thunderbird's database
+            AutoconfigSource {
+                provider: AutoconfigProvider::Mozilla,
+                url: format!("https://autoconfig.thunderbird.net/v1.1/{}", domain),
+            },
+        ]
+    }
+
+    async fn fetch(&self, ctx: &Context, param: &LoginParam) -> Result<Vec<ServerParams>> {
+        let params = match self.provider {
+            AutoconfigProvider::Mozilla => moz_autoconfigure(ctx, &self.url, &param).await?,
+            AutoconfigProvider::Outlook => outlk_autodiscover(ctx, &self.url).await?,
+        };
+
+        Ok(params)
+    }
+}
+
+/// Retrieve available autoconfigurations.
+///
+/// A Search configurations from the domain used in the email-address, prefer encrypted
+/// B. If we have no configuration yet, search configuration in Thunderbird's centeral database
+async fn get_autoconfig(
+    ctx: &Context,
+    param: &LoginParam,
+    param_domain: &str,
+    param_addr_urlencoded: &str,
+    progress: &impl Progress,
+) -> Option<Vec<ServerParams>> {
+    let sources = AutoconfigSource::all(param_domain, param_addr_urlencoded);
+
+    let mut p = 300;
+    for source in &sources {
+        let res = source.fetch(ctx, param).await;
+        progress.p(p);
+        p += 10;
+        if let Ok(res) = res {
+            return Some(res);
+        }
+    }
+
+    None
+}
+
+fn get_offline_autoconfig(context: &Context, addr: &str) -> Option<Vec<ServerParams>> {
+    info!(
+        context,
+        "checking internal provider-info for offline autoconfig"
+    );
+
+    if let Some(provider) = provider::get_provider_info(&addr) {
+        match provider.status {
+            provider::Status::OK | provider::Status::PREPARATION => {
+                if provider.server.is_empty() {
+                    info!(context, "offline autoconfig found, but no servers defined");
+                    None
+                } else {
+                    info!(context, "offline autoconfig found");
+                    let servers = provider
+                        .server
+                        .iter()
+                        .map(|s| ServerParams {
+                            protocol: s.protocol,
+                            socket: s.socket,
+                            hostname: s.hostname.to_string(),
+                            port: s.port,
+                            username: match s.username_pattern {
+                                UsernamePattern::EMAIL => addr.to_string(),
+                                UsernamePattern::EMAILLOCALPART => {
+                                    if let Some(at) = addr.find('@') {
+                                        addr.split_at(at).0.to_string()
+                                    } else {
+                                        addr.to_string()
+                                    }
+                                }
+                            },
+                        })
+                        .collect();
+                    Some(servers)
+                }
+            }
+            provider::Status::BROKEN => {
+                info!(context, "offline autoconfig found, provider is broken");
+                None
+            }
+        }
+    } else {
+        info!(context, "no offline autoconfig found");
+        None
+    }
+}
+
+async fn try_imap_one_param(
+    context: &Context,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
+    imap: &mut Imap,
+) -> Result<(), ConfigurationError> {
+    let inf = format!(
+        "imap: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+    );
+    info!(context, "Trying: {}", inf);
+
+    if let Err(err) = imap.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
+    } else {
+        info!(context, "success: {}", inf);
+        Ok(())
+    }
+}
+
+async fn try_smtp_one_param(
+    context: &Context,
+    param: &ServerLoginParam,
+    addr: &str,
+    oauth2: bool,
+    smtp: &mut Smtp,
+) -> Result<(), ConfigurationError> {
+    let inf = format!(
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
+    );
+    info!(context, "Trying: {}", inf);
+
+    if let Err(err) = smtp.connect(context, param, addr, oauth2).await {
+        info!(context, "failure: {}", err);
+        Err(ConfigurationError {
+            config: inf,
+            msg: err.to_string(),
+        })
+    } else {
+        info!(context, "success: {}", inf);
+        smtp.disconnect().await;
+        Ok(())
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("Trying {config}…\nError: {msg}")]
+pub struct ConfigurationError {
+    config: String,
+    msg: String,
+}
+
+async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationError>) -> String {
+    let first_err = if let Some(f) = errors.first() {
+        f
+    } else {
+        return "".to_string();
+    };
+
+    if errors
+        .iter()
+        .all(|e| e.msg.to_lowercase().contains("could not resolve"))
+    {
+        return context
+            .stock_str(StockMessage::ErrorNoNetwork)
+            .await
+            .to_string();
+    }
+
+    if errors.iter().all(|e| e.msg == first_err.msg) {
+        return first_err.msg.to_string();
+    }
+
+    errors.iter().map(|e| e.to_string()).join("\n\n")
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("Invalid email address: {0:?}")]
+    InvalidEmailAddress(String),
+
+    #[error("XML error at position {position}: {error}")]
+    InvalidXml {
+        position: usize,
+        #[source]
+        error: quick_xml::Error,
+    },
+
+    #[error("Failed to get URL: {0}")]
+    ReadUrlError(#[from] self::read_url::Error),
+
+    #[error("Number of redirection is exceeded")]
+    RedirectionError,
+}
+
+#[cfg(test)]
+mod tests {
+    #![allow(clippy::indexing_slicing)]
+
+    use super::*;
+    use crate::config::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_no_panic_on_bad_credentials() {
+        let t = TestContext::new().await;
+        t.ctx
+            .set_config(Config::Addr, Some("probably@unexistant.addr"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::MailPw, Some("123456"))
+            .await
+            .unwrap();
+        assert!(t.ctx.configure().await.is_err());
+    }
+
+    #[async_std::test]
+    async fn test_get_offline_autoconfig() {
+        let context = TestContext::new().await.ctx;
+
+        let addr = "someone123@example.org";
+        assert!(get_offline_autoconfig(&context, addr).is_none());
+
+        let addr = "someone123@nauta.cu";
+        let found_params = get_offline_autoconfig(&context, addr).unwrap();
+        assert_eq!(found_params.len(), 2);
+        assert_eq!(found_params[0].protocol, Protocol::IMAP);
+        assert_eq!(found_params[0].hostname, "imap.nauta.cu".to_string());
+        assert_eq!(found_params[1].protocol, Protocol::SMTP);
+        assert_eq!(found_params[1].hostname, "smtp.nauta.cu".to_string());
+    }
+}

src/configure/server_params.rs

@@ -25,20 +25,16 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    fn expand_usernames(self, addr: &str) -> Vec<ServerParams> {
+    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
-            res.push(Self {
-                username: addr.to_string(),
-                ..self.clone()
-            });
+            self.username = addr.to_string();
+            res.push(self.clone());
 
             if let Some(at) = addr.find('@') {
-                res.push(Self {
-                    username: addr.split_at(at).0.to_string(),
-                    ..self
-                });
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
             }
         } else {
             res.push(self)
@@ -46,87 +42,75 @@ impl ServerParams {
         res
     }
 
-    fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
+    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
         if self.hostname.is_empty() {
-            vec![
-                Self {
-                    hostname: param_domain.to_string(),
-                    ..self.clone()
-                },
-                Self {
-                    hostname: match self.protocol {
-                        Protocol::Imap => "imap.".to_string() + param_domain,
-                        Protocol::Smtp => "smtp.".to_string() + param_domain,
-                    },
-                    ..self.clone()
-                },
-                Self {
-                    hostname: "mail.".to_string() + param_domain,
-                    ..self
-                },
-            ]
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::IMAP => "imap.".to_string() + param_domain,
+                Protocol::SMTP => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
         } else {
-            vec![self]
+            res.push(self);
         }
+        res
     }
 
-    fn expand_ports(mut self) -> Vec<ServerParams> {
+    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
         // Try to infer port from socket security.
         if self.port == 0 {
             self.port = match self.socket {
-                Socket::Ssl => match self.protocol {
-                    Protocol::Imap => 993,
-                    Protocol::Smtp => 465,
+                Socket::SSL => match self.protocol {
+                    Protocol::IMAP => 993,
+                    Protocol::SMTP => 465,
                 },
-                Socket::Starttls | Socket::Plain => match self.protocol {
-                    Protocol::Imap => 143,
-                    Protocol::Smtp => 587,
+                Socket::STARTTLS | Socket::Plain => match self.protocol {
+                    Protocol::IMAP => 143,
+                    Protocol::SMTP => 587,
                 },
                 Socket::Automatic => 0,
             }
         }
 
+        let mut res = Vec::new();
         if self.port == 0 {
             // Neither port nor security is set.
             //
             // Try common secure combinations.
 
-            vec![
-                // Try STARTTLS
-                Self {
-                    socket: Socket::Starttls,
-                    port: match self.protocol {
-                        Protocol::Imap => 143,
-                        Protocol::Smtp => 587,
-                    },
-                    ..self.clone()
-                },
-                // Try TLS
-                Self {
-                    socket: Socket::Ssl,
-                    port: match self.protocol {
-                        Protocol::Imap => 993,
-                        Protocol::Smtp => 465,
-                    },
-                    ..self
-                },
-            ]
+            // Try STARTTLS
+            self.socket = Socket::STARTTLS;
+            self.port = match self.protocol {
+                Protocol::IMAP => 143,
+                Protocol::SMTP => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::SSL;
+            self.port = match self.protocol {
+                Protocol::IMAP => 993,
+                Protocol::SMTP => 465,
+            };
+            res.push(self);
         } else if self.socket == Socket::Automatic {
-            vec![
-                // Try TLS over user-provided port.
-                Self {
-                    socket: Socket::Ssl,
-                    ..self.clone()
-                },
-                // Try STARTTLS over user-provided port.
-                Self {
-                    socket: Socket::Starttls,
-                    ..self
-                },
-            ]
+            // Try TLS over user-provided port.
+            self.socket = Socket::SSL;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::STARTTLS;
+            res.push(self);
         } else {
-            vec![self]
+            res.push(self);
         }
+        res
     }
 }
 
@@ -156,10 +140,10 @@ mod tests {
     fn test_expand_param_vector() {
         let v = expand_param_vector(
             vec![ServerParams {
-                protocol: Protocol::Imap,
+                protocol: Protocol::IMAP,
                 hostname: "example.net".to_string(),
                 port: 0,
-                socket: Socket::Ssl,
+                socket: Socket::SSL,
                 username: "foobar".to_string(),
             }],
             "foobar@example.net",
@@ -169,44 +153,12 @@ mod tests {
         assert_eq!(
             v,
             vec![ServerParams {
-                protocol: Protocol::Imap,
+                protocol: Protocol::IMAP,
                 hostname: "example.net".to_string(),
                 port: 993,
-                socket: Socket::Ssl,
+                socket: Socket::SSL,
                 username: "foobar".to_string(),
             }],
         );
-
-        let v = expand_param_vector(
-            vec![ServerParams {
-                protocol: Protocol::Smtp,
-                hostname: "example.net".to_string(),
-                port: 123,
-                socket: Socket::Automatic,
-                username: "foobar".to_string(),
-            }],
-            "foobar@example.net",
-            "example.net",
-        );
-
-        assert_eq!(
-            v,
-            vec![
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string()
-                },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Starttls,
-                    username: "foobar".to_string()
-                }
-            ],
-        );
     }
 }

src/constants.rs

@@ -1,11 +1,11 @@
-//! # Constants.
-use deltachat_derive::{FromSql, ToSql};
-use once_cell::sync::Lazy;
+//! # Constants
+use deltachat_derive::*;
+use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 
-use crate::chat::ChatId;
-
-pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
+lazy_static! {
+    pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
+}
 
 #[derive(
     Debug,
@@ -21,11 +21,11 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
     Serialize,
     Deserialize,
 )]
-#[repr(i8)]
+#[repr(u8)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
-    Request = 2,
+    Deaddrop = 2,
 }
 
 impl Default for Blocked {
@@ -34,9 +34,7 @@ impl Default for Blocked {
     }
 }
 
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
@@ -50,9 +48,7 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum MediaQuality {
     Balanced = 0,
@@ -65,9 +61,7 @@ impl Default for MediaQuality {
     }
 }
 
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum KeyGenType {
     Default = 0,
@@ -81,9 +75,7 @@ impl Default for KeyGenType {
     }
 }
 
-#[derive(
-    Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i8)]
 pub enum VideochatType {
     Unknown = 0,
@@ -109,10 +101,9 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
-pub const DC_GCM_INFO_ONLY: u32 = 0x02;
 
-pub const DC_GCL_VERIFIED_ONLY: u32 = 0x01;
-pub const DC_GCL_ADD_SELF: u32 = 0x02;
+pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
+pub const DC_GCL_ADD_SELF: usize = 0x02;
 
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
@@ -123,14 +114,16 @@ pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 // do not use too small value that will annoy users checking for nonexistant updates.
 pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
 
+/// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
+pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
-pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3);
+pub const DC_CHAT_ID_TRASH: u32 = 3;
 /// only an indicator in a chatlist
-pub const DC_CHAT_ID_ARCHIVED_LINK: ChatId = ChatId::new(6);
+pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
-pub const DC_CHAT_ID_ALLDONE_HINT: ChatId = ChatId::new(7);
+pub const DC_CHAT_ID_ALLDONE_HINT: u32 = 7;
 /// larger chat IDs are "real" chats, their messages are "real" messages.
-pub const DC_CHAT_ID_LAST_SPECIAL: ChatId = ChatId::new(9);
+pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
 
 #[derive(
     Debug,
@@ -152,7 +145,7 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
-    Mailinglist = 140,
+    VerifiedGroup = 130,
 }
 
 impl Default for Chattype {
@@ -165,18 +158,10 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// String that indicates that something is left out or truncated.
-pub const DC_ELLIPSIS: &str = "[...]";
-
-/// Message length limit.
-///
-/// To keep bubbles and chat flow usable and to avoid problems with controls using very long texts,
-/// we limit the text length to `DC_DESIRED_TEXT_LEN`.  If the text is longer, the full text can be
-/// retrieved using has_html()/get_html().
-///
-/// Note that for simplicity maximum length is defined as the number of Unicode Scalar Values (Rust
-/// `char`s), not Unicode Grapheme Clusters.
-pub const DC_DESIRED_TEXT_LEN: usize = 5000;
+/// approx. max. length returned by dc_msg_get_text()
+pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
+/// approx. max. length returned by dc_get_msg_info()
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -214,8 +199,7 @@ pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
 pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
-pub const BALANCED_AVATAR_SIZE: u32 = 256;
-pub const WORSE_AVATAR_SIZE: u32 = 128;
+pub const AVATAR_SIZE: u32 = 192;
 
 // max. width/height of images
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
@@ -224,11 +208,6 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
-// if more recipients are needed in SMTP's `RCPT TO:` header, recipient-list is splitted to chunks.
-// this does not affect MIME'e `To:` header.
-// can be overwritten by the setting `max_smtp_rcpt_to` in provider-db.
-pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
-
 #[derive(
     Debug,
     Display,
@@ -243,7 +222,7 @@ pub const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
     Serialize,
     Deserialize,
 )]
-#[repr(u32)]
+#[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
 
@@ -302,6 +281,16 @@ impl Default for Viewtype {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn derive_display_works_as_expected() {
+        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
+    }
+}
+
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
@@ -310,100 +299,3 @@ pub enum KeyType {
     Public = 0,
     Private = 1,
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use num_traits::FromPrimitive;
-
-    #[test]
-    fn test_derive_display_works_as_expected() {
-        assert_eq!(format!("{}", Viewtype::Audio), "Audio");
-    }
-
-    #[test]
-    fn test_viewtype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Viewtype::Unknown, Viewtype::default());
-        assert_eq!(Viewtype::Unknown, Viewtype::from_i32(0).unwrap());
-        assert_eq!(Viewtype::Text, Viewtype::from_i32(10).unwrap());
-        assert_eq!(Viewtype::Image, Viewtype::from_i32(20).unwrap());
-        assert_eq!(Viewtype::Gif, Viewtype::from_i32(21).unwrap());
-        assert_eq!(Viewtype::Sticker, Viewtype::from_i32(23).unwrap());
-        assert_eq!(Viewtype::Audio, Viewtype::from_i32(40).unwrap());
-        assert_eq!(Viewtype::Voice, Viewtype::from_i32(41).unwrap());
-        assert_eq!(Viewtype::Video, Viewtype::from_i32(50).unwrap());
-        assert_eq!(Viewtype::File, Viewtype::from_i32(60).unwrap());
-        assert_eq!(
-            Viewtype::VideochatInvitation,
-            Viewtype::from_i32(70).unwrap()
-        );
-    }
-
-    #[test]
-    fn test_chattype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Chattype::Undefined, Chattype::default());
-        assert_eq!(Chattype::Undefined, Chattype::from_i32(0).unwrap());
-        assert_eq!(Chattype::Single, Chattype::from_i32(100).unwrap());
-        assert_eq!(Chattype::Group, Chattype::from_i32(120).unwrap());
-        assert_eq!(Chattype::Mailinglist, Chattype::from_i32(140).unwrap());
-    }
-
-    #[test]
-    fn test_keygentype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(KeyGenType::Default, KeyGenType::default());
-        assert_eq!(KeyGenType::Default, KeyGenType::from_i32(0).unwrap());
-        assert_eq!(KeyGenType::Rsa2048, KeyGenType::from_i32(1).unwrap());
-        assert_eq!(KeyGenType::Ed25519, KeyGenType::from_i32(2).unwrap());
-    }
-
-    #[test]
-    fn test_keytype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(KeyType::Public, KeyType::from_i32(0).unwrap());
-        assert_eq!(KeyType::Private, KeyType::from_i32(1).unwrap());
-    }
-
-    #[test]
-    fn test_showemails_values() {
-        // values may be written to disk and must not change
-        assert_eq!(ShowEmails::Off, ShowEmails::default());
-        assert_eq!(ShowEmails::Off, ShowEmails::from_i32(0).unwrap());
-        assert_eq!(
-            ShowEmails::AcceptedContacts,
-            ShowEmails::from_i32(1).unwrap()
-        );
-        assert_eq!(ShowEmails::All, ShowEmails::from_i32(2).unwrap());
-    }
-
-    #[test]
-    fn test_blocked_values() {
-        // values may be written to disk and must not change
-        assert_eq!(Blocked::Not, Blocked::default());
-        assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
-        assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
-        assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
-    }
-
-    #[test]
-    fn test_mediaquality_values() {
-        // values may be written to disk and must not change
-        assert_eq!(MediaQuality::Balanced, MediaQuality::default());
-        assert_eq!(MediaQuality::Balanced, MediaQuality::from_i32(0).unwrap());
-        assert_eq!(MediaQuality::Worse, MediaQuality::from_i32(1).unwrap());
-    }
-
-    #[test]
-    fn test_videochattype_values() {
-        // values may be written to disk and must not change
-        assert_eq!(VideochatType::Unknown, VideochatType::default());
-        assert_eq!(VideochatType::Unknown, VideochatType::from_i32(0).unwrap());
-        assert_eq!(
-            VideochatType::BasicWebrtc,
-            VideochatType::from_i32(1).unwrap()
-        );
-        assert_eq!(VideochatType::Jitsi, VideochatType::from_i32(2).unwrap());
-    }
-}

src/context.rs

@@ -1,31 +1,27 @@
-//! Context module.
+//! Context module
 
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
 use std::ops::Deref;
-use std::time::{Instant, SystemTime};
 
-use anyhow::{bail, ensure, Result};
-use async_std::{
-    channel::{self, Receiver, Sender},
-    path::{Path, PathBuf},
-    sync::{Arc, Mutex, RwLock},
-    task,
-};
+use async_std::path::{Path, PathBuf};
+use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
-use crate::chat::{get_chat_cnt, ChatId};
+use crate::chat::*;
 use crate::config::Config;
-use crate::constants::DC_VERSION_STR;
-use crate::contact::Contact;
-use crate::dc_tools::{duration_to_str, time};
+use crate::constants::*;
+use crate::contact::*;
+use crate::dc_tools::duration_to_str;
+use crate::error::*;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{self, MessageState, MsgId};
-use crate::quota::QuotaInfo;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
+use std::time::SystemTime;
 
 #[derive(Clone, Debug)]
 pub struct Context {
@@ -48,7 +44,7 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: Bob,
+    pub(crate) bob: RwLock<Bob>,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -63,16 +59,7 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Recently loaded quota information, if any.
-    /// Set to `None` if quota was never tried to load.
-    pub(crate) quota: RwLock<Option<QuotaInfo>>,
-
-    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
-
-    /// ID for this `Context` in the current process.
-    ///
-    /// This allows for multiple `Context`s open in a single process where each context can
-    /// be identified by this ID.
+    /// Id for this context on the current device.
     pub(crate) id: u32,
 
     creation_time: SystemTime,
@@ -96,7 +83,6 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
-    res.insert("num_cpus", num_cpus::get().to_string());
     res.insert("level", "awesome".into());
     res
 }
@@ -135,7 +121,7 @@ impl Context {
             os_name: Some(os_name),
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(),
-            bob: Default::default(),
+            bob: RwLock::new(Default::default()),
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
@@ -144,9 +130,7 @@ impl Context {
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
-            quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
-            last_full_folder_scan: Mutex::new(None),
         };
 
         let ctx = Context {
@@ -160,22 +144,29 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        if self.inner.is_io_running().await {
+        if self.is_io_running().await {
             info!(self, "IO is already running");
             return;
         }
 
         {
             let l = &mut *self.inner.scheduler.write().await;
-            if let Err(err) = l.start(self.clone()).await {
-                error!(self, "Failed to start IO: {}", err)
-            }
+            l.start(self.clone()).await;
         }
     }
 
+    /// Returns if the IO scheduler is running.
+    pub async fn is_io_running(&self) -> bool {
+        self.inner.is_io_running().await
+    }
+
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
 
         self.inner.stop_io().await;
     }
@@ -206,10 +197,7 @@ impl Context {
         });
     }
 
-    /// Returns a receiver for emitted events.
-    ///
-    /// Multiple emitters can be created, but note that in this case each emitted event will
-    /// only be received by one of the emitters, not by all of them.
+    /// Get the next queued event.
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
@@ -231,7 +219,7 @@ impl Context {
 
         s.ongoing_running = true;
         s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel::bounded(1);
+        let (sender, receiver) = channel(1);
         s.cancel_sender = Some(sender);
 
         Ok(receiver)
@@ -258,9 +246,7 @@ impl Context {
         let s_a = &self.running_state;
         let mut s = s_a.write().await;
         if let Some(cancel) = s.cancel_sender.take() {
-            if let Err(err) = cancel.send(()).await {
-                warn!(self, "could not cancel ongoing: {:?}", err);
-            }
+            cancel.send(()).await;
         }
 
         if s.ongoing_running && !s.shall_stop_ongoing {
@@ -279,72 +265,67 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub async fn get_info(&self) -> Result<BTreeMap<&'static str, String>> {
+    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await?;
-        let l2 = LoginParam::from_database(self, "configured_").await?;
-        let displayname = self.get_config(Config::Displayname).await?;
-        let chats = get_chat_cnt(self).await? as usize;
-        let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
-        let request_msgs = message::get_request_msg_cnt(self).await as usize;
-        let contacts = Contact::get_real_cnt(self).await? as usize;
-        let is_configured = self.get_config_int(Config::Configured).await?;
-        let socks5_enabled = self.get_config_int(Config::Socks5Enabled).await?;
+        let l = LoginParam::from_database(self, "").await;
+        let l2 = LoginParam::from_database(self, "configured_").await;
+        let displayname = self.get_config(Config::Displayname).await;
+        let chats = get_chat_cnt(self).await as usize;
+        let real_msgs = message::get_real_msg_cnt(self).await as usize;
+        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
+        let contacts = Contact::get_real_cnt(self).await as usize;
+        let is_configured = self.get_config_int(Config::Configured).await;
         let dbversion = self
             .sql
-            .get_raw_config_int("dbversion")
-            .await?
+            .get_raw_config_int(self, "dbversion")
+            .await
             .unwrap_or_default();
         let journal_mode = self
             .sql
-            .query_get_value("PRAGMA journal_mode;", paramsv![])
-            .await?
+            .query_get_value(self, "PRAGMA journal_mode;", paramsv![])
+            .await
             .unwrap_or_else(|| "unknown".to_string());
-        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await?;
-        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
-        let bcc_self = self.get_config_int(Config::BccSelf).await?;
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await;
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await;
+        let bcc_self = self.get_config_int(Config::BccSelf).await;
 
-        let prv_key_cnt = self
+        let prv_key_cnt: Option<isize> = self
             .sql
-            .count("SELECT COUNT(*) FROM keypairs;", paramsv![])
-            .await?;
+            .query_get_value(self, "SELECT COUNT(*) FROM keypairs;", paramsv![])
+            .await;
 
-        let pub_key_cnt = self
+        let pub_key_cnt: Option<isize> = self
             .sql
-            .count("SELECT COUNT(*) FROM acpeerstates;", paramsv![])
-            .await?;
+            .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
+            .await;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
             Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch).await?;
-        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await?;
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await?;
-        let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
-        let sentbox_move = self.get_config_int(Config::SentboxMove).await?;
+        let inbox_watch = self.get_config_int(Config::InboxWatch).await;
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
+        let mvbox_move = self.get_config_int(Config::MvboxMove).await;
         let folders_configured = self
             .sql
-            .get_raw_config_int("folders_configured")
-            .await?
+            .get_raw_config_int(self, "folders_configured")
+            .await
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
             .get_config(Config::ConfiguredSentboxFolder)
-            .await?
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
             .get_config(Config::ConfiguredMvboxFolder)
-            .await?
+            .await
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
-
-        // insert values
-        res.insert("bot", self.get_config_int(Config::Bot).await?.to_string());
         res.insert("number_of_chats", chats.to_string());
-        res.insert("number_of_chat_messages", unblocked_msgs.to_string());
-        res.insert("messages_in_contact_requests", request_msgs.to_string());
+        res.insert("number_of_chat_messages", real_msgs.to_string());
+        res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
@@ -354,91 +335,41 @@ impl Context {
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
-                .await?
+                .await
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
-        res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
-        res.insert(
-            "fetch_existing_msgs",
-            self.get_config_int(Config::FetchExistingMsgs)
-                .await?
-                .to_string(),
-        );
-        res.insert(
-            "show_emails",
-            self.get_config_int(Config::ShowEmails).await?.to_string(),
-        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
-        res.insert("sentbox_move", sentbox_move.to_string());
         res.insert("folders_configured", folders_configured.to_string());
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
-        res.insert(
-            "key_gen_type",
-            self.get_config_int(Config::KeyGenType).await?.to_string(),
-        );
         res.insert("bcc_self", bcc_self.to_string());
-        res.insert("private_key_count", prv_key_cnt.to_string());
-        res.insert("public_key_count", pub_key_cnt.to_string());
+        res.insert(
+            "private_key_count",
+            prv_key_cnt.unwrap_or_default().to_string(),
+        );
+        res.insert(
+            "public_key_count",
+            pub_key_cnt.unwrap_or_default().to_string(),
+        );
         res.insert("fingerprint", fingerprint_str);
-        res.insert(
-            "webrtc_instance",
-            self.get_config(Config::WebrtcInstance)
-                .await?
-                .unwrap_or_else(|| "<unset>".to_string()),
-        );
-        res.insert(
-            "media_quality",
-            self.get_config_int(Config::MediaQuality).await?.to_string(),
-        );
-        res.insert(
-            "delete_device_after",
-            self.get_config_int(Config::DeleteDeviceAfter)
-                .await?
-                .to_string(),
-        );
-        res.insert(
-            "delete_server_after",
-            self.get_config_int(Config::DeleteServerAfter)
-                .await?
-                .to_string(),
-        );
-        res.insert(
-            "last_housekeeping",
-            self.get_config_int(Config::LastHousekeeping)
-                .await?
-                .to_string(),
-        );
-        res.insert(
-            "scan_all_folders_debounce_secs",
-            self.get_config_int(Config::ScanAllFoldersDebounceSecs)
-                .await?
-                .to_string(),
-        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
 
-        Ok(res)
+        res
     }
 
-    /// Get a list of fresh, unmuted messages in unblocked chats.
-    ///
-    /// The list starts with the most recent message
-    /// and is typically used to show notifications.
-    /// Moreover, the number of returned messages
-    /// can be used for a badge counter on the app icon.
-    pub async fn get_fresh_msgs(&self) -> Result<Vec<MsgId>> {
-        let list = self
-            .sql
+    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
+        let show_deaddrop: i32 = 0;
+        self.sql
             .query_map(
                 concat!(
                     "SELECT m.id",
@@ -449,41 +380,67 @@ impl Context {
                     "        ON m.chat_id=c.id",
                     " WHERE m.state=?",
                     "   AND m.hidden=0",
-                    "   AND m.chat_id>9",
+                    "   AND m.chat_id>?",
                     "   AND ct.blocked=0",
-                    "   AND c.blocked=0",
-                    "   AND NOT(c.muted_until=-1 OR c.muted_until>?)",
+                    "   AND (c.blocked=0 OR c.blocked=?)",
                     " ORDER BY m.timestamp DESC,m.id DESC;"
                 ),
-                paramsv![MessageState::InFresh, time()],
+                paramsv![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
                 |row| row.get::<_, MsgId>(0),
                 |rows| {
-                    let mut list = Vec::new();
+                    let mut ret = Vec::new();
                     for row in rows {
-                        list.push(row?);
+                        ret.push(row?);
                     }
-                    Ok(list)
+                    Ok(ret)
                 },
             )
-            .await?;
-        Ok(list)
+            .await
+            .unwrap_or_default()
     }
 
-    /// Searches for messages containing the query string.
-    ///
-    /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
-    /// is `None` this searches messages from all chats.
-    pub async fn search_msgs(&self, chat_id: Option<ChatId>, query: &str) -> Result<Vec<MsgId>> {
-        let real_query = query.trim();
+    #[allow(non_snake_case)]
+    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+        let real_query = query.as_ref().trim();
         if real_query.is_empty() {
-            return Ok(Vec::new());
+            return Vec::new();
         }
-        let str_like_in_text = format!("%{}%", real_query);
+        let strLikeInText = format!("%{}%", real_query);
+        let strLikeBeg = format!("{}%", real_query);
 
-        let do_query = |query, params| {
-            self.sql.query_map(
+        let query = if !chat_id.is_unset() {
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                "   AND (txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp,m.id;"
+            )
+        } else {
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " LEFT JOIN chats c",
+                "        ON m.chat_id=c.id",
+                " WHERE m.chat_id>9",
+                "   AND m.hidden=0",
+                "   AND (c.blocked=0 OR c.blocked=?)",
+                "   AND ct.blocked=0",
+                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp DESC,m.id DESC;"
+            )
+        };
+
+        self.sql
+            .query_map(
                 query,
-                params,
+                paramsv![chat_id, strLikeInText, strLikeBeg],
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -493,72 +450,23 @@ impl Context {
                     Ok(ret)
                 },
             )
-        };
-
-        let list = if let Some(chat_id) = chat_id {
-            do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
-                 FROM msgs m
-                 LEFT JOIN contacts ct
-                        ON m.from_id=ct.id
-                 WHERE m.chat_id=?
-                   AND m.hidden=0
-                   AND ct.blocked=0
-                   AND txt LIKE ?
-                 ORDER BY m.timestamp,m.id;",
-                paramsv![chat_id, str_like_in_text],
-            )
-            .await?
-        } else {
-            // For performance reasons results are sorted only by `id`, that is in the order of
-            // message reception.
-            //
-            // Unlike chat view, sorting by `timestamp` is not necessary but slows down the query by
-            // ~25% according to benchmarks.
-            //
-            // To speed up incremental search, where queries for few characters usually return lots
-            // of unwanted results that are discarded moments later, we added `LIMIT 1000`.
-            // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
-            // The limit is documented and UI may add a hint when getting 1000 results.
-            do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
-                 FROM msgs m
-                 LEFT JOIN contacts ct
-                        ON m.from_id=ct.id
-                 LEFT JOIN chats c
-                        ON m.chat_id=c.id
-                 WHERE m.chat_id>9
-                   AND m.hidden=0
-                   AND c.blocked=0
-                   AND ct.blocked=0
-                   AND m.txt LIKE ?
-                 ORDER BY m.id DESC LIMIT 1000",
-                paramsv![str_like_in_text],
-            )
-            .await?
-        };
-
-        Ok(list)
+            .await
+            .unwrap_or_default()
     }
 
-    pub async fn is_inbox(&self, folder_name: &str) -> Result<bool> {
-        let inbox = self.get_config(Config::ConfiguredInboxFolder).await?;
-        Ok(inbox.as_deref() == Some(folder_name))
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn is_sentbox(&self, folder_name: &str) -> Result<bool> {
-        let sentbox = self.get_config(Config::ConfiguredSentboxFolder).await?;
-        Ok(sentbox.as_deref() == Some(folder_name))
+    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
-        let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
-        Ok(mvbox.as_deref() == Some(folder_name))
-    }
-
-    pub async fn is_spam_folder(&self, folder_name: &str) -> Result<bool> {
-        let spam = self.get_config(Config::ConfiguredSpamFolder).await?;
-        Ok(spam.as_deref() == Some(folder_name))
+    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
@@ -567,13 +475,6 @@ impl Context {
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
-
-    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
-        let mut wal_fname = OsString::new();
-        wal_fname.push(dbfile.file_name().unwrap_or_default());
-        wal_fname.push("-wal");
-        dbfile.with_file_name(wal_fname)
-    }
 }
 
 impl InnerContext {
@@ -582,19 +483,14 @@ impl InnerContext {
     }
 
     async fn stop_io(&self) {
-        if self.is_io_running().await {
-            let token = {
-                let lock = &*self.scheduler.read().await;
-                lock.pre_stop().await
-            };
-            {
-                let lock = &mut *self.scheduler.write().await;
-                lock.stop(token).await;
-            }
-        }
-
-        if let Some(ephemeral_task) = self.ephemeral_task.write().await.take() {
-            ephemeral_task.cancel().await;
+        assert!(self.is_io_running().await, "context is already stopped");
+        let token = {
+            let lock = &*self.scheduler.read().await;
+            lock.pre_stop().await
+        };
+        {
+            let lock = &mut *self.scheduler.write().await;
+            lock.stop(token).await;
         }
     }
 }
@@ -617,16 +513,7 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::chat::{
-        get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, ChatId, MuteDuration,
-    };
-    use crate::constants::{Viewtype, DC_CONTACT_ID_SELF};
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::dc_tools::dc_create_outgoing_rfc724_mid;
-    use crate::message::Message;
-    use crate::test_utils::TestContext;
-    use std::time::Duration;
-    use strum::IntoEnumIterator;
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_wrong_db() {
@@ -640,146 +527,10 @@ mod tests {
     #[async_std::test]
     async fn test_get_fresh_msgs() {
         let t = TestContext::new().await;
-        let fresh = t.get_fresh_msgs().await.unwrap();
+        let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
 
-    async fn receive_msg(t: &TestContext, chat: &Chat) {
-        let members = get_chat_contacts(t, chat.id).await.unwrap();
-        let contact = Contact::load_from_db(t, *members.first().unwrap())
-            .await
-            .unwrap();
-        let msg = format!(
-            "From: {}\n\
-             To: alice@example.com\n\
-             Message-ID: <{}>\n\
-             Chat-Version: 1.0\n\
-             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-             \n\
-             hello\n",
-            contact.get_addr(),
-            dc_create_outgoing_rfc724_mid(None, contact.get_addr())
-        );
-        println!("{}", msg);
-        dc_receive_imf(t, msg.as_bytes(), "INBOX", 1, false)
-            .await
-            .unwrap();
-    }
-
-    #[async_std::test]
-    async fn test_get_fresh_msgs_and_muted_chats() {
-        // receive various mails in 3 chats
-        let t = TestContext::new_alice().await;
-        let bob = t.create_chat_with_contact("", "bob@g.it").await;
-        let claire = t.create_chat_with_contact("", "claire@g.it").await;
-        let dave = t.create_chat_with_contact("", "dave@g.it").await;
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        receive_msg(&t, &bob).await;
-        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.unwrap().len(), 1);
-        assert_eq!(bob.id.get_fresh_msg_cnt(&t).await.unwrap(), 1);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        receive_msg(&t, &claire).await;
-        receive_msg(&t, &claire).await;
-        assert_eq!(
-            get_chat_msgs(&t, claire.id, 0, None).await.unwrap().len(),
-            2
-        );
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
-
-        receive_msg(&t, &dave).await;
-        receive_msg(&t, &dave).await;
-        receive_msg(&t, &dave).await;
-        assert_eq!(get_chat_msgs(&t, dave.id, 0, None).await.unwrap().len(), 3);
-        assert_eq!(dave.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
-
-        // mute one of the chats
-        set_muted(&t, claire.id, MuteDuration::Forever)
-            .await
-            .unwrap();
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
-
-        // receive more messages
-        receive_msg(&t, &bob).await;
-        receive_msg(&t, &claire).await;
-        receive_msg(&t, &dave).await;
-        assert_eq!(
-            get_chat_msgs(&t, claire.id, 0, None).await.unwrap().len(),
-            3
-        );
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
-
-        // unmute claire again
-        set_muted(&t, claire.id, MuteDuration::NotMuted)
-            .await
-            .unwrap();
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
-    }
-
-    #[async_std::test]
-    async fn test_get_fresh_msgs_and_muted_until() {
-        let t = TestContext::new_alice().await;
-        let bob = t.create_chat_with_contact("", "bob@g.it").await;
-        receive_msg(&t, &bob).await;
-        assert_eq!(get_chat_msgs(&t, bob.id, 0, None).await.unwrap().len(), 1);
-
-        // chat is unmuted by default, here and in the following assert(),
-        // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
-        // have the same view to the database.
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        // test get_fresh_msgs() with mute_until in the future
-        set_muted(
-            &t,
-            bob.id,
-            MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
-        )
-        .await
-        .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        // to test get_fresh_msgs() with mute_until in the past,
-        // we need to modify the database directly
-        t.sql
-            .execute(
-                "UPDATE chats SET muted_until=? WHERE id=?;",
-                paramsv![time() - 3600, bob.id],
-            )
-            .await
-            .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        // test get_fresh_msgs() with "forever" mute_until
-        set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        // to test get_fresh_msgs() with invalid mute_until (everything < -1),
-        // that results in "muted forever" by definition.
-        t.sql
-            .execute(
-                "UPDATE chats SET muted_until=-2 WHERE id=?;",
-                paramsv![bob.id],
-            )
-            .await
-            .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-    }
-
     #[async_std::test]
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
@@ -835,14 +586,14 @@ mod tests {
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
         let t = TestContext::new().await;
-        std::mem::drop(t);
+        std::mem::drop(t.ctx);
     }
 
     #[async_std::test]
     async fn test_get_info() {
         let t = TestContext::new().await;
 
-        let info = t.get_info().await.unwrap();
+        let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());
     }
 
@@ -853,141 +604,4 @@ mod tests {
         assert!(info.get("database_dir").is_none());
         assert_eq!(info.get("level").unwrap(), "awesome");
     }
-
-    #[async_std::test]
-    async fn test_get_info_completeness() {
-        // For easier debugging,
-        // get_info() shall return all important information configurable by the Config-values.
-        //
-        // There are exceptions for Config-values considered to be unimportant,
-        // too sensitive or summarized in another item.
-        let skip_from_get_info = vec![
-            "addr",
-            "displayname",
-            "imap_certificate_checks",
-            "mail_server",
-            "mail_user",
-            "mail_pw",
-            "mail_port",
-            "mail_security",
-            "notify_about_wrong_pw",
-            "save_mime_headers",
-            "selfstatus",
-            "send_server",
-            "send_user",
-            "send_pw",
-            "send_port",
-            "send_security",
-            "server_flags",
-            "smtp_certificate_checks",
-            "socks5_host",
-            "socks5_port",
-            "socks5_user",
-            "socks5_password",
-        ];
-        let t = TestContext::new().await;
-        let info = t.get_info().await.unwrap();
-        for key in Config::iter() {
-            let key: String = key.to_string();
-            if !skip_from_get_info.contains(&&*key)
-                && !key.starts_with("configured")
-                && !key.starts_with("sys.")
-            {
-                assert!(
-                    info.contains_key(&*key),
-                    "'{}' missing in get_info() output",
-                    key
-                );
-            }
-        }
-    }
-
-    #[async_std::test]
-    async fn test_search_msgs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, DC_CONTACT_ID_SELF).await?;
-        let chat = alice
-            .create_chat_with_contact("Bob", "bob@example.org")
-            .await;
-
-        // Global search finds nothing.
-        let res = alice.search_msgs(None, "foo").await?;
-        assert!(res.is_empty());
-
-        // Search in chat with Bob finds nothing.
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert!(res.is_empty());
-
-        // Add messages to chat with Bob.
-        let mut msg1 = Message::new(Viewtype::Text);
-        msg1.set_text(Some("foobar".to_string()));
-        send_msg(&alice, chat.id, &mut msg1).await?;
-
-        let mut msg2 = Message::new(Viewtype::Text);
-        msg2.set_text(Some("barbaz".to_string()));
-        send_msg(&alice, chat.id, &mut msg2).await?;
-
-        // Global search with a part of text finds the message.
-        let res = alice.search_msgs(None, "ob").await?;
-        assert_eq!(res.len(), 1);
-
-        // Global search for "bar" matches both "foobar" and "barbaz".
-        let res = alice.search_msgs(None, "bar").await?;
-        assert_eq!(res.len(), 2);
-
-        // Message added later is returned first.
-        assert_eq!(res.get(0), Some(&msg2.id));
-        assert_eq!(res.get(1), Some(&msg1.id));
-
-        // Global search with longer text does not find any message.
-        let res = alice.search_msgs(None, "foobarbaz").await?;
-        assert!(res.is_empty());
-
-        // Search for random string finds nothing.
-        let res = alice.search_msgs(None, "abc").await?;
-        assert!(res.is_empty());
-
-        // Search in chat with Bob finds the message.
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert_eq!(res.len(), 1);
-
-        // Search in Saved Messages does not find the message.
-        let res = alice.search_msgs(Some(self_talk), "foo").await?;
-        assert!(res.is_empty());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_limit_search_msgs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let chat = alice
-            .create_chat_with_contact("Bob", "bob@example.org")
-            .await;
-
-        // Add 999 messages
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("foobar".to_string()));
-        for _ in 0..999 {
-            send_msg(&alice, chat.id, &mut msg).await?;
-        }
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 999);
-
-        // Add one more message, no limit yet
-        send_msg(&alice, chat.id, &mut msg).await?;
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 1000);
-
-        // Add one more message, that one is truncated then
-        send_msg(&alice, chat.id, &mut msg).await?;
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 1000);
-
-        // In-chat should not be not limited
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert_eq!(res.len(), 1001);
-
-        Ok(())
-    }
 }

src/dc_tools.rs

@@ -2,34 +2,129 @@
 //! no references to Context and other "larger" entities here.
 
 use core::cmp::{max, min};
-use std::borrow::Cow;
 use std::fmt;
 use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
+use std::{borrow::Cow, sync::Arc};
 
-use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
+use async_std::{
+    path::{Path, PathBuf},
+    sync::Mutex,
+    task,
+};
+use async_trait::async_trait;
 
-use anyhow::{bail, Error};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
+use crate::error::{bail, Error};
 use crate::events::EventType;
 use crate::message::Message;
 use crate::provider::get_provider_update_timestamp;
-use crate::stock_str;
+use crate::stock::StockMessage;
+
+#[derive(Debug)]
+pub struct ProgressHandlerInner<F: Fn(usize) + Send> {
+    progress_limit: usize,
+    emitted_progress: f64,
+    step_fraction: f64,
+    f: F,
+}
+
+#[derive(Debug)]
+pub struct ProgressHandler<F: 'static + Fn(usize) + Send> {
+    inner: Arc<Mutex<ProgressHandlerInner<F>>>,
+}
+
+impl<F> ProgressHandler<F>
+where
+    F: 'static + Fn(usize) + Send,
+{
+    /// If step_fraction is e.g. 15, then every 100ms we will step by 1/15th of the remaining interval.
+    /// The bigger this value, the slower the progress bar will move in the beginning.
+    /// f is the function that is invoked when progress is made.
+    pub fn new(step_fraction: f64, f: F) -> Self {
+        let ret = Arc::new(Mutex::new(ProgressHandlerInner {
+            progress_limit: 1,
+            emitted_progress: 0f64,
+            step_fraction,
+            f,
+        }));
+        let cloned = ret.clone();
+        task::spawn(async move {
+            loop {
+                task::sleep(Duration::from_millis(100)).await;
+                {
+                    let mut lock = cloned.lock().await;
+                    let limit = lock.progress_limit;
+                    if limit == 1000 || limit == 0 {
+                        return;
+                    }
+                    let last = lock.emitted_progress;
+
+                    let next = last + ((limit as f64 - last) / lock.step_fraction);
+
+                    if (next / 10f64).ceil() - (last / 10f64).ceil() > 0f64 {
+                        (lock.f)(next.ceil() as usize);
+                    }
+                    lock.emitted_progress = next;
+
+                    drop(lock);
+                };
+            }
+        });
+        Self { inner: ret }
+    }
+}
+
+#[async_trait]
+pub trait Progress {
+    /// Report actually made progress 0-1000 promille. The progress bar will slowly move toward the value set by this function.
+    /// Set rather high values as the progress bar will stay lower first,
+    /// i.e. don't start with values near 0 and end with values near 1000
+    fn p(&self, progress: usize);
+    /// Stops the progress handler without emitting any other events
+    async fn kill(&self);
+}
+
+#[async_trait]
+impl<F> Progress for ProgressHandler<F>
+where
+    F: 'static + Fn(usize) + Send,
+{
+    fn p(&self, progress: usize) {
+        assert!(
+            progress <= 1000,
+            "value in range 0..1000 expected with: 0=error, 1..999=progress, 1000=success"
+        );
+        let inner = self.inner.clone();
+        task::spawn(async move {
+            if progress == 1000 || progress == 0 {
+                let inner = &inner.lock().await;
+                (inner.f)(progress);
+            }
+            inner.lock().await.progress_limit = progress;
+        });
+    }
+    async fn kill(&self) {
+        self.inner.lock().await.progress_limit = 0usize;
+    }
+}
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
+    let ellipse = "[...]";
+
     let count = buf.chars().count();
-    if count > approx_chars + DC_ELLIPSIS.len() {
+    if approx_chars > 0 && count > approx_chars + ellipse.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -37,15 +132,40 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSIS))
+            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSIS))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
         }
     } else {
         Cow::Borrowed(buf)
     }
 }
 
+/// the colors must fulfill some criterions as:
+/// - contrast to black and to white
+/// - work as a text-color
+/// - being noticeable on a typical map
+/// - harmonize together while being different enough
+/// (therefore, we cannot just use random rgb colors :)
+const COLORS: [u32; 16] = [
+    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
+    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
+];
+
+#[allow(clippy::indexing_slicing)]
+pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
+    let str_lower = s.as_ref().to_lowercase();
+    let mut checksum = 0;
+    let bytes = str_lower.as_bytes();
+    for (i, byte) in bytes.iter().enumerate() {
+        checksum += (i + 1) * *byte as usize;
+        checksum %= 0x00ff_ffff;
+    }
+    let color_index = checksum % COLORS.len();
+
+    COLORS[color_index]
+}
+
 /* ******************************************************************************
  * date/time tools
  ******************************************************************************/
@@ -142,14 +262,15 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
     if now < known_past_timestamp {
         let mut msg = Message::new(Viewtype::Text);
         msg.text = Some(
-            stock_str::bad_time_msg_body(
-                context,
-                Local
-                    .timestamp(now, 0)
-                    .format("%Y-%m-%d %H:%M:%S")
-                    .to_string(),
-            )
-            .await,
+            context
+                .stock_string_repl_str(
+                    StockMessage::BadTimeMsgBody,
+                    Local
+                        .timestamp(now, 0)
+                        .format("%Y-%m-%d %H:%M:%S")
+                        .to_string(),
+                )
+                .await,
         );
         add_device_msg_with_importance(
             context,
@@ -173,7 +294,12 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam
 async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
     if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
         let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_str::update_reminder_msg_body(context).await);
+        msg.text = Some(
+            context
+                .stock_str(StockMessage::UpdateReminderMsgBody)
+                .await
+                .into(),
+        );
         add_device_msg(
             context,
             Some(
@@ -504,15 +630,34 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
+pub(crate) async fn get_next_backup_path_old(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<PathBuf, Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut path = folder.clone();
+        path.push(format!("{}-{}.bak", stem, i));
+        if !path.exists().await {
+            return Ok(path);
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 /// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
 /// it can be renamed to dest_path. This guarantees that the backup is complete.
-pub(crate) async fn get_next_backup_path(
+pub(crate) async fn get_next_backup_path_new(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<(PathBuf, PathBuf), Error> {
     let folder = PathBuf::from(folder.as_ref());
     let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
-        // Don't change this file name format, in has_backup() we use string comparison to determine which backup is newer:
         .format("delta-chat-backup-%Y-%m-%d")
         .to_string();
 
@@ -608,8 +753,19 @@ impl FromStr for EmailAddress {
                 if local.is_empty() {
                     return err("empty string is not valid for local part");
                 }
-                if domain.is_empty() {
-                    return err("missing domain after '@'");
+                if domain.len() <= 3 {
+                    return err("domain is too short");
+                }
+                let dot = domain.find('.');
+                match dot {
+                    None => {
+                        return err("invalid domain");
+                    }
+                    Some(dot_idx) => {
+                        if dot_idx >= domain.len() - 2 {
+                            return err("invalid domain");
+                        }
+                    }
                 }
                 Ok(EmailAddress {
                     local: (*local).to_string(),
@@ -629,9 +785,17 @@ impl rusqlite::types::ToSql for EmailAddress {
     }
 }
 
+/// Utility to check if a in the binary represantion of listflags
+/// the bit at position bitindex is 1.
+pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
+    let listflags = listflags as usize;
+    (listflags & bitindex) == bitindex
+}
+
 /// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
-pub(crate) fn improve_single_line_input(input: &str) -> String {
+pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
     input
+        .as_ref()
         .replace("\n", " ")
         .replace("\r", " ")
         .trim()
@@ -646,28 +810,11 @@ where
     T: AsRef<str>,
 {
     fn is_none_or_empty(&self) -> bool {
-        !matches!(self, Some(s) if !s.as_ref().is_empty())
-    }
-}
-
-pub fn remove_subject_prefix(last_subject: &str) -> String {
-    let subject_start = if last_subject.starts_with("Chat:") {
-        0
-    } else {
-        // "Antw:" is the longest abbreviation in
-        // <https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages>,
-        // so look at the first _5_ characters:
-        match last_subject.chars().take(5).position(|c| c == ':') {
-            Some(prefix_end) => prefix_end + 1,
-            None => 0,
+        match self {
+            Some(s) if !s.as_ref().is_empty() => false,
+            _ => true,
         }
-    };
-    last_subject
-        .chars()
-        .skip(subject_start)
-        .collect::<String>()
-        .trim()
-        .to_string()
+    }
 }
 
 #[cfg(test)]
@@ -675,8 +822,10 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use std::convert::TryInto;
 
-    use crate::test_utils::TestContext;
+    use crate::constants::*;
+    use crate::test_utils::*;
 
     #[test]
     fn test_rust_ftoa() {
@@ -711,7 +860,10 @@ mod tests {
         assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
 
         assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
-        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
+        assert_eq!(
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
+            "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
+        );
 
         // 9 characters, so no truncation
         assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
@@ -811,19 +963,12 @@ mod tests {
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!(
-            "user@localhost".parse::<EmailAddress>().unwrap(),
-            EmailAddress {
-                local: "user".into(),
-                domain: "localhost".into()
-            }
-        );
         assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
         assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
-        assert!("tt.dd@uu".parse::<EmailAddress>().is_ok());
-        assert!("u@d".parse::<EmailAddress>().is_ok());
-        assert!("u@d.".parse::<EmailAddress>().is_ok());
-        assert!("u@d.t".parse::<EmailAddress>().is_ok());
+        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
             "u@d.tt".parse::<EmailAddress>().unwrap(),
             EmailAddress {
@@ -831,7 +976,7 @@ mod tests {
                 domain: "d.tt".into(),
             }
         );
-        assert!("u@tt".parse::<EmailAddress>().is_ok());
+        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
@@ -869,7 +1014,7 @@ mod tests {
     #[async_std::test]
     async fn test_file_handling() {
         let t = TestContext::new().await;
-        let context = &t;
+        let context = &t.ctx;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
                 $ctx.get_blobdir()
@@ -879,6 +1024,16 @@ mod tests {
         }
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
+        if dc_file_exist!(context, "$BLOBDIR/foobar").await
+            || dc_file_exist!(context, "$BLOBDIR/dada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
+            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
+        {
+            dc_delete_file(context, "$BLOBDIR/foobar").await;
+            dc_delete_file(context, "$BLOBDIR/dada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
+            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
+        }
         assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
             .await
             .is_ok());
@@ -921,15 +1076,29 @@ mod tests {
         assert!(!dc_file_exist!(context, &fn0).await);
     }
 
+    #[test]
+    fn test_listflags_has() {
+        let listflags: u32 = 0x1101;
+        assert!(listflags_has(listflags, 0x1));
+        assert!(!listflags_has(listflags, 0x10));
+        assert!(listflags_has(listflags, 0x100));
+        assert!(listflags_has(listflags, 0x1000));
+        let listflags: u32 = (DC_GCL_ADD_SELF | DC_GCL_VERIFIED_ONLY).try_into().unwrap();
+        assert!(listflags_has(listflags, DC_GCL_VERIFIED_ONLY));
+        assert!(listflags_has(listflags, DC_GCL_ADD_SELF));
+        let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
+        assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
+    }
+
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
         let t = TestContext::new().await;
         assert_ne!(
-            dc_create_smeared_timestamp(&t).await,
-            dc_create_smeared_timestamp(&t).await
+            dc_create_smeared_timestamp(&t.ctx).await,
+            dc_create_smeared_timestamp(&t.ctx).await
         );
         assert!(
-            dc_create_smeared_timestamp(&t).await
+            dc_create_smeared_timestamp(&t.ctx).await
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -941,13 +1110,13 @@ mod tests {
     async fn test_create_smeared_timestamps() {
         let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t, count as usize).await;
-        let next = dc_smeared_time(&t).await;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t, count as usize).await;
-        let next = dc_smeared_time(&t).await;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
+        let next = dc_smeared_time(&t.ctx).await;
         assert!((start + count - 1) < next);
     }
 
@@ -1021,61 +1190,53 @@ mod tests {
             / 1_000;
 
         // a correct time must not add a device message
-        maybe_warn_on_bad_time(&t, timestamp_now, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // we cannot find out if a date in the future is wrong - a device message is not added
-        maybe_warn_on_bad_time(&t, timestamp_future, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // a date in the past must add a device message
-        maybe_warn_on_bad_time(&t, timestamp_past, get_provider_update_timestamp()).await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // the message should be added only once a day - test that an hour later and nearly a day later
         maybe_warn_on_bad_time(
-            &t,
+            &t.ctx,
             timestamp_past + 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         maybe_warn_on_bad_time(
-            &t,
+            &t.ctx,
             timestamp_past + 60 * 60 * 24 - 1,
             get_provider_update_timestamp(),
         )
         .await;
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // next day, there should be another device message
         maybe_warn_on_bad_time(
-            &t,
+            &t.ctx,
             timestamp_past + 60 * 60 * 24,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         assert_eq!(device_chat_id, chats.get_chat_id(0));
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 2);
     }
 
@@ -1087,66 +1248,51 @@ mod tests {
         // in about 6 months, the app should not be outdated
         // (if this fails, provider-db is not updated since 6 months)
         maybe_warn_on_outdated(
-            &t,
+            &t.ctx,
             timestamp_now + 180 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 0);
 
         // in 1 year, the app should be considered as outdated
         maybe_warn_on_outdated(
-            &t,
+            &t.ctx,
             timestamp_now + 365 * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
         assert_eq!(msgs.len(), 1);
 
         // do not repeat the warning every day ...
-        // (we test that for the 2 subsequent days, this may be the next month, so the result should be 1 or 2 device message)
         maybe_warn_on_outdated(
-            &t,
+            &t.ctx,
             timestamp_now + (365 + 1) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        maybe_warn_on_outdated(
-            &t,
-            timestamp_now + (365 + 2) * 24 * 60 * 60,
-            get_provider_update_timestamp(),
-        )
-        .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
-        let test_len = msgs.len();
-        assert!(test_len == 1 || test_len == 2);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
 
         // ... but every month
-        // (forward generous 33 days to avoid being in the same month as in the previous check)
         maybe_warn_on_outdated(
-            &t,
-            timestamp_now + (365 + 33) * 24 * 60 * 60,
+            &t.ctx,
+            timestamp_now + (365 + 31) * 24 * 60 * 60,
             get_provider_update_timestamp(),
         )
         .await;
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
         assert_eq!(chats.len(), 1);
         let device_chat_id = chats.get_chat_id(0);
-        let msgs = chat::get_chat_msgs(&t, device_chat_id, 0, None)
-            .await
-            .unwrap();
-        assert_eq!(msgs.len(), test_len + 1);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
     }
 }

src/dehtml.rs

@@ -1,55 +1,21 @@
-//! De-HTML.
+//! De-HTML
 //!
 //! A module to remove HTML tags from the email text
 
-use std::io::BufRead;
+use lazy_static::lazy_static;
+use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
-use once_cell::sync::Lazy;
-use quick_xml::{
-    events::{BytesEnd, BytesStart, BytesText},
-    Reader,
-};
-
-static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
+lazy_static! {
+    static ref LINE_RE: regex::Regex = regex::Regex::new(r"(\r?\n)+").unwrap();
+}
 
 struct Dehtml {
     strbuilder: String,
     add_text: AddText,
     last_href: Option<String>,
-    /// GMX wraps a quote in `<div name="quote">`. After a `<div name="quote">`, this count is
-    /// increased at each `<div>` and decreased at each `</div>`. This way we know when the quote ends.
-    /// If this is > `0`, then we are inside a `<div name="quote">`
-    divs_since_quote_div: u32,
-    /// Everything between <div name="quote"> and <div name="quoted-content"> is usually metadata
-    /// If this is > `0`, then we are inside a `<div name="quoted-content">`.
-    divs_since_quoted_content_div: u32,
-    /// All-Inkl just puts the quote into `<blockquote> </blockquote>`. This count is
-    /// increased at each `<blockquote>` and decreased at each `</blockquote>`.
-    blockquotes_since_blockquote: u32,
 }
 
-impl Dehtml {
-    fn line_prefix(&self) -> &str {
-        if self.divs_since_quoted_content_div > 0 || self.blockquotes_since_blockquote > 0 {
-            "> "
-        } else {
-            ""
-        }
-    }
-    fn append_prefix(&self, line_end: impl AsRef<str>) -> String {
-        // line_end is e.g. "\n\n". We add "> " if necessary.
-        line_end.as_ref().to_owned() + self.line_prefix()
-    }
-    fn get_add_text(&self) -> AddText {
-        if self.divs_since_quote_div > 0 && self.divs_since_quoted_content_div == 0 {
-            AddText::No // Everything between <div name="quoted"> and <div name="quoted_content"> is metadata which we don't want
-        } else {
-            self.add_text
-        }
-    }
-}
-
-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(Debug, PartialEq)]
 enum AddText {
     No,
     YesRemoveLineEnds,
@@ -58,28 +24,25 @@ enum AddText {
 
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
-pub fn dehtml(buf: &str) -> Option<String> {
+pub fn dehtml(buf: &str) -> String {
     let s = dehtml_quick_xml(buf);
     if !s.trim().is_empty() {
-        return Some(s);
+        return s;
     }
     let s = dehtml_manually(buf);
     if !s.trim().is_empty() {
-        return Some(s);
+        return s;
     }
-    None
+    buf.to_string()
 }
 
-fn dehtml_quick_xml(buf: &str) -> String {
+pub fn dehtml_quick_xml(buf: &str) -> String {
     let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
         add_text: AddText::YesRemoveLineEnds,
         last_href: None,
-        divs_since_quote_div: 0,
-        divs_since_quoted_content_div: 0,
-        blockquotes_since_blockquote: 0,
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
@@ -118,16 +81,13 @@ fn dehtml_quick_xml(buf: &str) -> String {
 }
 
 fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
-        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
+    if dehtml.add_text == AddText::YesPreserveLineEnds
+        || dehtml.add_text == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
+        if dehtml.add_text == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
-        } else if !dehtml.line_prefix().is_empty() {
-            let l = dehtml.append_prefix("\n");
-            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -135,16 +95,13 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
 }
 
 fn dehtml_cdata_cb(event: &BytesText, dehtml: &mut Dehtml) {
-    if dehtml.get_add_text() == AddText::YesPreserveLineEnds
-        || dehtml.get_add_text() == AddText::YesRemoveLineEnds
+    if dehtml.add_text == AddText::YesPreserveLineEnds
+        || dehtml.add_text == AddText::YesRemoveLineEnds
     {
         let last_added = escaper::decode_html_buf_sloppy(event.escaped()).unwrap_or_default();
 
-        if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
+        if dehtml.add_text == AddText::YesRemoveLineEnds {
             dehtml.strbuilder += LINE_RE.replace_all(&last_added, "\r").as_ref();
-        } else if !dehtml.line_prefix().is_empty() {
-            let l = dehtml.append_prefix("\n");
-            dehtml.strbuilder += LINE_RE.replace_all(&last_added, l.as_str()).as_ref();
         } else {
             dehtml.strbuilder += &last_added;
         }
@@ -155,15 +112,8 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "table" | "td" | "style" | "script" | "title" | "pre" => {
-            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
-            dehtml.add_text = AddText::YesRemoveLineEnds;
-        }
-        "div" => {
-            pop_tag(&mut dehtml.divs_since_quote_div);
-            pop_tag(&mut dehtml.divs_since_quoted_content_div);
-
-            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+        "p" | "div" | "table" | "td" | "style" | "script" | "title" | "pre" => {
+            dehtml.strbuilder += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "a" => {
@@ -174,16 +124,11 @@ fn dehtml_endtag_cb(event: &BytesEnd, dehtml: &mut Dehtml) {
             }
         }
         "b" | "strong" => {
-            if dehtml.get_add_text() != AddText::No {
-                dehtml.strbuilder += "*";
-            }
+            dehtml.strbuilder += "*";
         }
         "i" | "em" => {
-            if dehtml.get_add_text() != AddText::No {
-                dehtml.strbuilder += "_";
-            }
+            dehtml.strbuilder += "_";
         }
-        "blockquote" => pop_tag(&mut dehtml.blockquotes_since_blockquote),
         _ => {}
     }
 }
@@ -196,27 +141,19 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     match tag.as_str() {
-        "p" | "table" | "td" => {
-            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
-            dehtml.add_text = AddText::YesRemoveLineEnds;
-        }
-        #[rustfmt::skip]
-        "div" => {
-            maybe_push_tag(event, reader, "quote", &mut dehtml.divs_since_quote_div);
-            maybe_push_tag(event, reader, "quoted-content", &mut dehtml.divs_since_quoted_content_div);
-
-            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+        "p" | "div" | "table" | "td" => {
+            dehtml.strbuilder += "\n\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "br" => {
-            dehtml.strbuilder += &dehtml.append_prefix("\n");
+            dehtml.strbuilder += "\n";
             dehtml.add_text = AddText::YesRemoveLineEnds;
         }
         "style" | "script" | "title" => {
             dehtml.add_text = AddText::No;
         }
         "pre" => {
-            dehtml.strbuilder += &dehtml.append_prefix("\n\n");
+            dehtml.strbuilder += "\n\n";
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
@@ -237,52 +174,15 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             }
         }
         "b" | "strong" => {
-            if dehtml.get_add_text() != AddText::No {
-                dehtml.strbuilder += "*";
-            }
+            dehtml.strbuilder += "*";
         }
         "i" | "em" => {
-            if dehtml.get_add_text() != AddText::No {
-                dehtml.strbuilder += "_";
-            }
+            dehtml.strbuilder += "_";
         }
-        "blockquote" => dehtml.blockquotes_since_blockquote += 1,
         _ => {}
     }
 }
 
-/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
-/// The `counts`s are stored in the `Dehtml` struct.
-fn pop_tag(count: &mut u32) {
-    if *count > 0 {
-        *count -= 1;
-    }
-}
-
-/// In order to know when a specific tag is closed, we need to count the opening and closing tags.
-/// The `counts`s are stored in the `Dehtml` struct.
-fn maybe_push_tag(
-    event: &BytesStart,
-    reader: &Reader<impl BufRead>,
-    tag_name: &str,
-    count: &mut u32,
-) {
-    if *count > 0 || tag_contains_attr(event, reader, tag_name) {
-        *count += 1;
-    }
-}
-
-fn tag_contains_attr(event: &BytesStart, reader: &Reader<impl BufRead>, name: &str) -> bool {
-    event.attributes().any(|r| {
-        r.map(|a| {
-            a.unescape_and_decode_value(reader)
-                .map(|v| v == name)
-                .unwrap_or(false)
-        })
-        .unwrap_or(false)
-    })
-}
-
 pub fn dehtml_manually(buf: &str) -> String {
     // Just strip out everything between "<" and ">"
     let mut strbuilder = String::new();
@@ -314,7 +214,6 @@ mod tests {
                 "[ Foo ](https://example.com)",
             ),
             ("<b> bar </b>", "* bar *"),
-            ("<i>foo</i>", "_foo_"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
             // Despite missing ', this should be shown:
@@ -323,23 +222,21 @@ mod tests {
                 "<a href='https://get.delta.chat/'/>",
                 "[](https://get.delta.chat/)",
             ),
+            ("", ""),
             ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
             // Invalid html (at least DC should show the text if the html is invalid):
             ("<!some invalid html code>\n<b>some text</b>", "some text"),
+            ("<This text is in brackets>", "<This text is in brackets>"),
         ];
         for (input, output) in cases {
-            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
-        }
-        let none_cases = vec!["<html> </html>", ""];
-        for input in none_cases {
-            assert_eq!(dehtml(input), None);
+            assert_eq!(simplify(dehtml(input), true).0, output);
         }
     }
 
     #[test]
     fn test_dehtml_parse_br() {
         let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
-        let plain = dehtml(html).unwrap();
+        let plain = dehtml(html);
 
         assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
@@ -347,7 +244,7 @@ mod tests {
     #[test]
     fn test_dehtml_parse_href() {
         let html = "<a href=url>text</a";
-        let plain = dehtml(html).unwrap();
+        let plain = dehtml(html);
 
         assert_eq!(plain, "[text](url)");
     }
@@ -355,7 +252,7 @@ mod tests {
     #[test]
     fn test_dehtml_bold_text() {
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html).unwrap();
+        let plain = dehtml(html);
 
         assert_eq!(plain, "text *bold*<>");
     }
@@ -365,7 +262,7 @@ mod tests {
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let plain = dehtml(html).unwrap();
+        let plain = dehtml(html);
 
         assert_eq!(
             plain,
@@ -388,27 +285,7 @@ mod tests {
         </body>
         </html>
         "##;
-        let txt = dehtml(input).unwrap();
+        let txt = dehtml(input);
         assert_eq!(txt.trim(), "lots of text");
     }
-
-    #[test]
-    fn test_pre_tag() {
-        let input = "<html><pre>\ntwo\nlines\n</pre></html>";
-        let txt = dehtml(input).unwrap();
-        assert_eq!(txt.trim(), "two\nlines");
-    }
-
-    #[async_std::test]
-    async fn test_quote_div() {
-        let input = include_str!("../test-data/message/gmx-quote-body.eml");
-        let dehtml = dehtml(input).unwrap();
-        println!("{}", dehtml);
-        let (msg, forwarded, cut, top_quote, footer) = simplify(dehtml, false);
-        assert_eq!(msg, "Test");
-        assert_eq!(forwarded, false);
-        assert_eq!(cut, false);
-        assert_eq!(top_quote.as_deref(), Some("test"));
-        assert_eq!(footer, None);
-    }
 }

src/e2ee.rs

@@ -2,18 +2,18 @@
 
 use std::collections::HashSet;
 
-use anyhow::{bail, ensure, format_err, Result};
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
-use crate::aheader::{Aheader, EncryptPreference};
+use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
+use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
-use crate::keyring::Keyring;
-use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::keyring::*;
+use crate::peerstate::*;
 use crate::pgp;
 
 #[derive(Debug)]
@@ -26,9 +26,9 @@ pub struct EncryptHelper {
 impl EncryptHelper {
     pub async fn new(context: &Context) -> Result<EncryptHelper> {
         let prefer_encrypt =
-            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await?)
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await)
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await? {
+        let addr = match context.get_config(Config::ConfiguredAddr).await {
             None => {
                 bail!("addr not configured!");
             }
@@ -57,9 +57,12 @@ impl EncryptHelper {
     /// preferences, even if message copy is not sent to self.
     ///
     /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
-    /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
+    /// Autocrypt Level 1, version 1.1) and for messages sent in verified groups.
     ///
     /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
+    ///
+    /// Always returns `false` if one of the peerstates does not support Autocrypt (is in "reset"
+    /// state) or does not have a known key.
     pub fn should_encrypt(
         &self,
         context: &Context,
@@ -81,11 +84,7 @@ impl EncryptHelper {
                     match peerstate.prefer_encrypt {
                         EncryptPreference::NoPreference => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
-                        EncryptPreference::Reset => {
-                            if !e2ee_guaranteed {
-                                return Ok(false);
-                            }
-                        }
+                        EncryptPreference::Reset => return Ok(false),
                     };
                 }
                 None => {
@@ -113,7 +112,7 @@ impl EncryptHelper {
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate>, &str)>,
+        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
     ) -> Result<String> {
         let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
@@ -153,7 +152,7 @@ pub async fn try_decrypt(
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
+        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -163,10 +162,10 @@ pub async fn try_decrypt(
     // Apply Autocrypt header
     if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            peerstate.apply_header(header, message_time);
+            peerstate.apply_header(&header, message_time);
             peerstate.save_to_db(&context.sql, false).await?;
         } else {
-            let p = Peerstate::from_header(header, message_time);
+            let p = Peerstate::from_header(context, header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
@@ -178,9 +177,7 @@ pub async fn try_decrypt(
     let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
-        peerstate
-            .handle_fingerprint_change(context, message_time)
-            .await?;
+        peerstate.handle_fingerprint_change(context).await?;
         if let Some(key) = &peerstate.public_key {
             public_keyring_for_validate.add(key.clone());
         } else if let Some(key) = &peerstate.gossip_key {
@@ -211,74 +208,45 @@ pub async fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
-/// Returns a reference to the encrypted payload of a valid PGP/MIME message.
-///
-/// Returns `None` if the message is not a valid PGP/MIME message.
-fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
-    if mail.ctype.mimetype != "multipart/encrypted" {
-        return None;
-    }
+/// Returns a reference to the encrypted payload and validates the autocrypt structure.
+fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
+    ensure!(
+        mail.ctype.mimetype == "multipart/encrypted",
+        "Not a multipart/encrypted message: {}",
+        mail.ctype.mimetype
+    );
     if let [first_part, second_part] = &mail.subparts[..] {
-        if first_part.ctype.mimetype == "application/pgp-encrypted"
-            && second_part.ctype.mimetype == "application/octet-stream"
-        {
-            Some(second_part)
-        } else {
-            None
-        }
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
+
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
+
+        Ok(second_part)
     } else {
-        None
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
     }
 }
 
-/// Returns a reference to the encrypted payload of a ["Mixed
-/// Up"][pgpmime-message-mangling] message.
-///
-/// According to [RFC 3156] encrypted messages should have
-/// `multipart/encrypted` MIME type and two parts, but Microsoft
-/// Exchange and ProtonMail IMAP/SMTP Bridge are known to mangle this
-/// structure by changing the type to `multipart/mixed` and prepending
-/// an empty part at the start.
-///
-/// ProtonMail IMAP/SMTP Bridge prepends a part literally saying
-/// "Empty Message", so we don't check its contents at all, checking
-/// only for `text/plain` type.
-///
-/// Returns `None` if the message is not a "Mixed Up" message.
-///
-/// [RFC 3156]: https://www.rfc-editor.org/info/rfc3156
-/// [pgpmime-message-mangling]: https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html
-fn get_mixed_up_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
-    if mail.ctype.mimetype != "multipart/mixed" {
-        return None;
-    }
-    if let [first_part, second_part, third_part] = &mail.subparts[..] {
-        if first_part.ctype.mimetype == "text/plain"
-            && second_part.ctype.mimetype == "application/pgp-encrypted"
-            && third_part.ctype.mimetype == "application/octet-stream"
-        {
-            Some(third_part)
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-async fn decrypt_if_autocrypt_message(
+async fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
-    mail: &ParsedMail<'_>,
+    mail: &ParsedMail<'a>,
     private_keyring: Keyring<SignedSecretKey>,
     public_keyring_for_validate: Keyring<SignedPublicKey>,
     ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    let encrypted_data_part = match get_autocrypt_mime(mail).or_else(|| get_mixed_up_mime(mail)) {
-        None => {
+    let encrypted_data_part = match get_autocrypt_mime(mail) {
+        Err(_) => {
             // not an autocrypt mime message, abort and ignore
             return Ok(None);
         }
-        Some(res) => res,
+        Ok(res) => res,
     };
     info!(context, "Detected Autocrypt-mime message");
 
@@ -335,7 +303,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     false
 }
 
-/// Checks if a MIME structure contains a multipart/report part.
+/// Check if a MIME structure contains a multipart/report part.
 ///
 /// As reports are often unencrypted, we do not reset the Autocrypt header in
 /// this case.
@@ -360,7 +328,7 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
-        .await?
+        .await
         .ok_or_else(|| {
             format_err!(concat!(
                 "Failed to get self address, ",
@@ -377,10 +345,10 @@ mod tests {
 
     use crate::chat;
     use crate::constants::Viewtype;
+    use crate::contact::{Contact, Origin};
     use crate::message::Message;
     use crate::param::Param;
-    use crate::peerstate::ToSave;
-    use crate::test_utils::{bob_keypair, TestContext};
+    use crate::test_utils::*;
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -389,13 +357,13 @@ mod tests {
         async fn test_prexisting() {
             let t = TestContext::new().await;
             let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t).await.unwrap(), test_addr);
+            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
             let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t).await.is_err());
+            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }
 
@@ -442,12 +410,27 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[async_std::test]
-    async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
+    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
         let alice = TestContext::new_alice().await;
         let bob = TestContext::new_bob().await;
 
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
+        let (contact_alice_id, _modified) = Contact::add_or_lookup(
+            &bob.ctx,
+            "Alice",
+            "alice@example.com",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+
+        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
+        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
@@ -527,75 +510,4 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
         Ok(())
     }
-
-    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
-        let addr = "bob@foo.bar";
-        let pub_key = bob_keypair().public;
-        let peerstate = Peerstate {
-            addr: addr.into(),
-            last_seen: 13,
-            last_seen_autocrypt: 14,
-            prefer_encrypt,
-            public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint()),
-            gossip_key: Some(pub_key.clone()),
-            gossip_timestamp: 15,
-            gossip_key_fingerprint: Some(pub_key.fingerprint()),
-            verified_key: Some(pub_key.clone()),
-            verified_key_fingerprint: Some(pub_key.fingerprint()),
-            to_save: Some(ToSave::All),
-            fingerprint_changed: false,
-        };
-        vec![(Some(peerstate), addr)]
-    }
-
-    #[async_std::test]
-    async fn test_should_encrypt() {
-        let t = TestContext::new_alice().await;
-        let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
-
-        // test with EncryptPreference::NoPreference:
-        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
-        let ps = new_peerstates(EncryptPreference::NoPreference);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
-
-        // test with EncryptPreference::Reset
-        let ps = new_peerstates(EncryptPreference::Reset);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
-
-        // test with EncryptPreference::Mutual (self is also Mutual)
-        let ps = new_peerstates(EncryptPreference::Mutual);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
-
-        // test with missing peerstate
-        let ps = vec![(None, "bob@foo.bar")];
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
-    }
-
-    #[test]
-    fn test_mixed_up_mime() -> Result<()> {
-        // "Mixed Up" mail as received when sending an encrypted
-        // message using Delta Chat Desktop via ProtonMail IMAP/SMTP
-        // Bridge.
-        let mixed_up_mime = include_bytes!("../test-data/message/protonmail-mixed-up.eml");
-        let mail = mailparse::parse_mail(mixed_up_mime)?;
-        assert!(get_autocrypt_mime(&mail).is_none());
-        assert!(get_mixed_up_mime(&mail).is_some());
-
-        // Same "Mixed Up" mail repaired by Thunderbird 78.9.0.
-        //
-        // It added `X-Enigmail-Info: Fixed broken PGP/MIME message`
-        // header although the repairing is done by the built-in
-        // OpenPGP support, not Enigmail.
-        let repaired_mime = include_bytes!("../test-data/message/protonmail-repaired.eml");
-        let mail = mailparse::parse_mail(repaired_mime)?;
-        assert!(get_autocrypt_mime(&mail).is_some());
-        assert!(get_mixed_up_mime(&mail).is_none());
-
-        Ok(())
-    }
 }

src/ephemeral.rs

@@ -1,4 +1,4 @@
-//! # Ephemeral messages.
+//! # Ephemeral messages
 //!
 //! Ephemeral messages are messages that have an Ephemeral-Timer
 //! header attached to them, which specifies time in seconds after
@@ -56,26 +56,24 @@
 //! the database entries which are expired either according to their
 //! ephemeral message timers or global `delete_server_after` setting.
 
-use std::convert::{TryFrom, TryInto};
-use std::num::ParseIntError;
-use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
-
-use anyhow::{ensure, Context as _, Result};
-use async_std::task;
-use serde::{Deserialize, Serialize};
-
-use crate::chat::{send_msg, ChatId};
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
 use crate::constants::{
     Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
 };
 use crate::context::Context;
 use crate::dc_tools::time;
+use crate::error::{ensure, Error};
 use crate::events::EventType;
-use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
-use crate::stock_str;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -147,10 +145,10 @@ impl rusqlite::types::FromSql for Timer {
 
 impl ChatId {
     /// Get ephemeral message timer value in seconds.
-    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer> {
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
         let timer = context
             .sql
-            .query_get_value(
+            .query_get_value_result(
                 "SELECT ephemeral_timer FROM chats WHERE id=?;",
                 paramsv![self],
             )
@@ -166,7 +164,7 @@ impl ChatId {
         self,
         context: &Context,
         timer: Timer,
-    ) -> Result<()> {
+    ) -> Result<(), Error> {
         ensure!(!self.is_special(), "Invalid chat ID");
 
         context
@@ -189,7 +187,7 @@ impl ChatId {
     /// Set ephemeral message timer value in seconds.
     ///
     /// If timer value is 0, disable ephemeral message timer.
-    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<()> {
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
         if timer == self.get_ephemeral_timer(context).await? {
             return Ok(());
         }
@@ -213,58 +211,29 @@ pub(crate) async fn stock_ephemeral_timer_changed(
     timer: Timer,
     from_id: u32,
 ) -> String {
-    match timer {
-        Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await,
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
         Timer::Enabled { duration } => match duration {
-            0..=59 => {
-                stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await
-            }
-            60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await,
-            61..=3599 => {
-                stock_str::msg_ephemeral_timer_minutes(
-                    context,
-                    format!("{}", (f64::from(duration) / 6.0).round() / 10.0),
-                    from_id,
-                )
-                .await
-            }
-            3600 => stock_str::msg_ephemeral_timer_hour(context, from_id).await,
-            3601..=86399 => {
-                stock_str::msg_ephemeral_timer_hours(
-                    context,
-                    format!("{}", (f64::from(duration) / 360.0).round() / 10.0),
-                    from_id,
-                )
-                .await
-            }
-            86400 => stock_str::msg_ephemeral_timer_day(context, from_id).await,
-            86401..=604_799 => {
-                stock_str::msg_ephemeral_timer_days(
-                    context,
-                    format!("{}", (f64::from(duration) / 8640.0).round() / 10.0),
-                    from_id,
-                )
-                .await
-            }
-            604_800 => stock_str::msg_ephemeral_timer_week(context, from_id).await,
-            _ => {
-                stock_str::msg_ephemeral_timer_weeks(
-                    context,
-                    format!("{}", (f64::from(duration) / 60480.0).round() / 10.0),
-                    from_id,
-                )
-                .await
-            }
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
         },
-    }
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
 }
 
 impl MsgId {
     /// Returns ephemeral message timer value for the message.
-    pub(crate) async fn ephemeral_timer(self, context: &Context) -> anyhow::Result<Timer> {
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
         let res = match context
             .sql
-            .query_get_value(
+            .query_get_value_result(
                 "SELECT ephemeral_timer FROM msgs WHERE id=?",
                 paramsv![self],
             )
@@ -277,7 +246,7 @@ impl MsgId {
     }
 
     /// Starts ephemeral message timer for the message if it is not started yet.
-    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
             let ephemeral_timestamp = time() + i64::from(duration);
 
@@ -304,35 +273,30 @@ impl MsgId {
 /// false. This function does not emit the MsgsChanged event itself,
 /// because it is also called when chatlist is reloaded, and emitting
 /// MsgsChanged there will cause infinite reload loop.
-pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool> {
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
     let mut updated = context
         .sql
         .execute(
-            // If you change which information is removed here, also change MsgId::trash() and
-            // which information dc_receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-            r#"
-UPDATE msgs
-SET 
-  chat_id=?, txt='', subject='', txt_raw='', 
-  mime_headers='', from_id=0, to_id=0, param=''
-WHERE
-  ephemeral_timestamp != 0
-  AND ephemeral_timestamp <= ?
-  AND chat_id != ?
-"#,
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
             paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
         )
-        .await
-        .context("update failed")?
+        .await?
         > 0;
 
-    if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_SELF)
-            .await?
-            .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, DC_CONTACT_ID_DEVICE)
-            .await?
-            .unwrap_or_default();
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
 
         let threshold_timestamp = time() - delete_device_after;
 
@@ -357,8 +321,7 @@ WHERE
                     device_chat_id
                 ],
             )
-            .await
-            .context("deleted update failed")?;
+            .await?;
 
         updated |= rows_modified > 0;
     }
@@ -380,15 +343,13 @@ WHERE
 pub async fn schedule_ephemeral_task(context: &Context) {
     let ephemeral_timestamp: Option<i64> = match context
         .sql
-        .query_get_value(
-            r#"
-    SELECT ephemeral_timestamp
-    FROM msgs
-    WHERE ephemeral_timestamp != 0
-      AND chat_id != ?
-    ORDER BY ephemeral_timestamp ASC
-    LIMIT 1;
-    "#,
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
             paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
         )
         .await
@@ -442,10 +403,10 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 ///
 /// It looks up the trash chat too, to find messages that are already
 /// deleted locally, but not deleted on the server.
-pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
     let now = time();
 
-    let threshold_timestamp = match context.get_config_delete_server_after().await? {
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
         None => 0,
         Some(delete_server_after) => now - delete_server_after,
     };
@@ -456,16 +417,12 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Resul
             "SELECT id FROM msgs \
          WHERE ( \
          timestamp < ? \
-         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
          ) \
          AND server_uid != 0 \
-         AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
-            |row| {
-                let msg_id: MsgId = row.get(0)?;
-                Ok(msg_id)
-            },
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
         )
         .await
 }
@@ -479,7 +436,7 @@ pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Resul
 ///
 /// This function is supposed to be called in the background,
 /// e.g. from housekeeping task.
-pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
     context
         .sql
         .execute(
@@ -502,19 +459,12 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::param::Params;
-    use async_std::task::sleep;
-
     use super::*;
-    use crate::test_utils::TestContext;
-    use crate::{
-        chat::{self, Chat, ChatItem},
-        dc_tools::IsNoneOrEmpty,
-    };
+    use crate::test_utils::*;
 
     #[async_std::test]
     async fn test_stock_ephemeral_messages() {
-        let context = TestContext::new().await;
+        let context = TestContext::new().await.ctx;
 
         assert_eq!(
             stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
@@ -522,78 +472,24 @@ mod tests {
         );
 
         assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 1 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 1 s by me."
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 30 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 30 s by me."
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 60 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 1 minute by me."
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 90 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 1.5 minutes by me."
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
         );
         assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 30 * 60 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 30 minutes by me."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 60 * 60 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 1 hour by me."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 5400 },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 1.5 hours by me."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 2 * 60 * 60
-                },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 2 hours by me."
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -601,21 +497,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                0
             )
             .await,
-            "Message deletion timer is set to 1 day by me."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 2 * 24 * 60 * 60
-                },
-                DC_CONTACT_ID_SELF
-            )
-            .await,
-            "Message deletion timer is set to 2 days by me."
+            "Message deletion timer is set to 1 day."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -623,10 +508,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                0
             )
             .await,
-            "Message deletion timer is set to 1 week by me."
+            "Message deletion timer is set to 1 week."
         );
         assert_eq!(
             stock_ephemeral_timer_changed(
@@ -634,147 +519,10 @@ mod tests {
                 Timer::Enabled {
                     duration: 4 * 7 * 24 * 60 * 60
                 },
-                DC_CONTACT_ID_SELF
+                0
             )
             .await,
-            "Message deletion timer is set to 4 weeks by me."
+            "Message deletion timer is set to 4 weeks."
         );
     }
-
-    #[async_std::test]
-    async fn test_ephemeral_timer() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        // Alice sends message to Bob
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-
-        // Alice sends second message to Bob, with no timer
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled
-        );
-
-        // Bob sets ephemeral timer and sends a message about timer change
-        chat_bob
-            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        let sent_timer_change = bob.pop_sent_msg().await;
-
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Bob receives message from Alice.
-        // Alice message has no timer. However, Bob should not disable timer,
-        // because Alice replies to old message.
-        bob.recv_msg(&sent).await;
-
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Disabled
-        );
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Alice receives message from Bob
-        alice.recv_msg(&sent_timer_change).await;
-
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_ephemeral_delete_msgs() {
-        let t = TestContext::new_alice().await;
-        let chat = t.get_self_chat().await;
-
-        t.send_text(chat.id, "Saved message, which we delete manually")
-            .await;
-        let msg = t.get_last_msg_in(chat.id).await;
-        msg.id.delete_from_db(&t).await.unwrap();
-        check_msg_was_deleted(&t, &chat, msg.id).await;
-
-        chat.id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 1 })
-            .await
-            .unwrap();
-        let msg = t
-            .send_text(chat.id, "Saved message, disappearing after 1s")
-            .await;
-
-        sleep(Duration::from_millis(1100)).await;
-
-        // Check checks that the msg was deleted locally
-        check_msg_was_deleted(&t, &chat, msg.sender_msg_id).await;
-
-        // Check that the msg will be deleted on the server
-        // First of all, set a server_uid so that DC thinks that it's actually possible to delete
-        t.sql
-            .execute(
-                "UPDATE msgs SET server_uid=1 WHERE id=?",
-                paramsv![msg.sender_msg_id],
-            )
-            .await
-            .unwrap();
-        let job = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(
-            job,
-            Some(job::Job::new(
-                job::Action::DeleteMsgOnImap,
-                msg.sender_msg_id.to_u32(),
-                Params::new(),
-                0,
-            ))
-        );
-        // Let's assume that executing the job fails on first try and the job is saved to the db
-        job.unwrap().save(&t).await.unwrap();
-
-        // Make sure that we don't get yet another job when loading from db
-        let job2 = job::load_imap_deletion_job(&t).await.unwrap();
-        assert_eq!(job2, None);
-    }
-
-    async fn check_msg_was_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
-        let chat_items = chat::get_chat_msgs(t, chat.id, 0, None).await.unwrap();
-        // Check that the chat is empty except for possibly info messages:
-        for item in &chat_items {
-            if let ChatItem::Message { msg_id } = item {
-                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
-                assert!(msg.is_info())
-            }
-        }
-
-        // Check that if there is a message left, the text and metadata are gone
-        if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, 0);
-            assert_eq!(msg.to_id, 0);
-            assert!(msg.text.is_none_or_empty(), "{:?}", msg.text);
-            let rawtxt: Option<String> = t
-                .sql
-                .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", paramsv![msg_id])
-                .await
-                .unwrap();
-            assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
-        }
-    }
 }