stop timings

.circleci/config.yml

@@ -0,0 +1,68 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+            
+jobs:
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash scripts/remote_python_packaging.sh
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: ls -laR workspace
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      - remote_python_packaging:
+          filters:
+            branches:
+              only: master
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            branches:
+              only: master
+
+      - build_doxygen:
+          filters:
+            branches:
+              only: master

.github/dependabot.yml

@@ -1,9 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "cargo"
-    directory: "/"
-    schedule:
-      interval: "monthly"
-    commit-message:
-      prefix: "cargo"
-    open-pull-requests-limit: 50

.github/mergeable.yml

@@ -1,26 +0,0 @@
-version: 2
-mergeable:
-  - when: pull_request.*
-    name: "Changelog check"
-    validate:
-      - do: or
-        validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/**'
-                required: ['CHANGELOG.md']
-    fail:
-      - do: checks
-        status: 'action_required' 
-        payload:
-          title: Changlog might need an update
-          summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -8,9 +8,6 @@ on:
       - staging
       - trying
 
-env:
-  RUSTFLAGS: -Dwarnings
-
 jobs:
 
   fmt:
@@ -21,11 +18,9 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.50.0
           override: true
       - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/cargo@v1
         with:
           command: fmt
@@ -37,15 +32,13 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.50.0
             components: clippy
             override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches
+          args: --workspace --tests --examples
 
   docs:
     name: Rust doc comments
@@ -72,26 +65,26 @@ jobs:
 
   build_and_test:
     name: Build and test
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
     strategy:
       matrix:
-        include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
-          - os: ubuntu-latest
-            rust: 1.60.0
-            python: 3.9
-          - os: windows-latest
-            rust: 1.60.0
-            python: false # Python bindings compilation on Windows is not supported.
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
+        rust: [1.50.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
 
-          # Minimum Supported Rust Version = 1.56.0
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.56.0
-            python: 3.7
-    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master
 
@@ -101,43 +94,32 @@ jobs:
         toolchain: ${{ matrix.rust }}
         override: true
 
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
 
     - name: check
       uses: actions-rs/cargo@v1
       with:
-        command: check
-        args: --all --bins --examples --tests --features repl --benches
+        command:  check
+        args: --all --bins --examples --tests --features repl
 
     - name: tests
       uses: actions-rs/cargo@v1
       with:
         command: test
         args: --all
-
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
-
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
-
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3

.github/workflows/dependabot.yml

@@ -1,21 +0,0 @@
-name: Dependabot auto-approve
-on: pull_request
-
-permissions:
-  pull-requests: write
-
-jobs:
-  dependabot:
-    runs-on: ubuntu-latest
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - name: Dependabot metadata
-        id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-      - name: Approve a PR
-        run: gh pr review --approve "$PR_URL"
-        env:
-          PR_URL: ${{github.event.pull_request.html_url}}
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/remote_tests.yml

@@ -0,0 +1,21 @@
+name: Remote tests
+on: [push]
+jobs:
+  remote_tests_python:
+    name: Remote Python tests
+    runs-on: ubuntu-latest
+    env:
+      CIRCLE_BRANCH: ${{ github.ref }}
+      CIRCLE_JOB: remote_tests_python
+      CIRCLE_BUILD_NUM: ${{ github.run_number }}
+      DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+    steps:
+      - uses: actions/checkout@v2
+      - run: mkdir -m 700 -p ~/.ssh
+      - run: touch ~/.ssh/id_ed25519
+      - run: chmod 600 ~/.ssh/id_ed25519
+      - run: 'echo "$SSH_KEY" | base64 -d > ~/.ssh/id_ed25519'
+        shell: bash
+        env:
+          SSH_KEY: ${{ secrets.SSH_KEY }}
+      - run: scripts/remote_tests_python.sh

.github/workflows/repl.yml

@@ -1,32 +0,0 @@
-# Manually triggered action to build a Windows repl.exe which users can
-# download to debug complex bugs.
-
-name: Build Windows REPL .exe
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_repl:
-    name: Build REPL example
-    runs-on: windows-latest
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
-
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'

.github/workflows/upload-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on rs.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat --no-deps
-    - name: Upload to rs.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/rs/"
-

.github/workflows/upload-ffi-docs.yml

@@ -1,28 +0,0 @@
-name: Build & Deploy Documentation on cffi.delta.chat
-
-on:
-  push:
-    branches:
-      - master
-      - docs-gh-action
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - uses: actions-rs/toolchain@v1
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat_ffi --no-deps
-    - name: Upload to cffi.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/cffi/"
-

.gitignore

@@ -28,8 +28,3 @@ deltachat-ffi/xml
 .rsynclist
 
 coverage/
-.DS_Store
-.vscode/launch.json
-python/accounts.txt
-python/all-testaccounts.txt
-tmp/

CHANGELOG.md

@@ -1,553 +1,4 @@
-# Changelog
-
-## 1.79.0
-
-### API-Changes
-- add `dc_event_emitter_close()` to termitate event loops in a thread-safe way
-
-### Changes
-- Send locations in the background regardless of SMTP loop activity #3247
-- refactorings #3268
-- improve tests and ci #3266 #3271
-
-### Fixes
-- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
-- fix correct message escaping consisting of a dot in SMTP protocol #3265
-
-
-## 1.78.0
-
-### API-Changes
-- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
-- add `dc_resend_msgs()` #3238
-- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
-  this is done by `dc_provider_new_from_email_with_dns()` now #3256
-
-### Changes
-- introduce multiple self addresses with the "configured" address always being the primary one #2896
-- Further improve finding the correct server after logging in #3208
-- `get_connectivity_html()` returns HTML as non-scalable #3213
-- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
-- Speed up message receiving via IMAP a bit #3225
-- mark messages as seen on IMAP in batches #3223
-- remove Received: based draft detection heuristic #3230
-- Use pkgconfig for building Python package #2590
-- don't start io on unconfigured context #2664
-- do not assign group IDs to ad-hoc groups #2798
-- dynamic libraries use dylib extension on Darwin #3226
-- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
-- improve documentation #3214 #3220 #3237
-- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
-
-### Fixes
-- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
-- Fix a bug where a blocked contact could send a contact request #3218
-- Make sure, videochat-room-names are always URL-safe #3231
-- Try removing account folder multiple times in case of failure #3229
-- Ignore messages from all spam folders if there are many #3246
-- Hide location-only messages instead of displaying empty bubbles #3248
-
-
-## 1.77.0
-
-### API changes
-- change semantics of `dc_get_webxdc_status_updates()` second parameter
-  and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
-
-### Changes
-- add more SMTP logging #3093
-- place common headers like `From:` before the large `Autocrypt:` header #3079
-- keep track of securejoin joiner status in database to survive restarts #2920
-- remove never used `SentboxMove` option #3111
-- improve speed by caching config values #3131 #3145
-- optimize `markseen_msgs` #3141
-- automatically accept chats with outgoing messages #3143
-- `dc_receive_imf` refactorings #3154 #3156 #3159
-- add index to speedup deletion of expired ephemeral messages #3155
-- muted chats stay archived on new messages #3184
-- support `min_api` from Webxdc manifests #3206
-- do not read whole webxdc file into memory #3109
-- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
-- improve documentation #2983 #3112 #3103 #3118 #3120
-
-### Fixes
-- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
-- fix an issue where the app crashes when trying to export a backup #3195
-- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
-- do not delete messages without Message-IDs as duplicates #3095
-- assign replies from a different email address to the correct chat #3119
-- assing outgoing private replies to the correct chat #3177
-- start ephemeral timer when seen status is synchronized via IMAP #3122
-- do not create empty contact requests with "setup changed" messages;
-  instead, send a "setup changed" message into all chats we share with the peer #3187
-- do not delete duplicate messages on IMAP immediately to accidentally deleting
-  the last copy #3138
-- clear more columns when message expires due to `delete_device_after` setting #3181
-- do not try to use stale SMTP connections #3180
-- slightly improve finding the correct server after logging in #3207
-- retry message sending automatically if loop is not interrupted #3183
-- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
-
-
-## 1.76.0
-
-### Changes
-- move messages in batches #3058
-- delete messages in batches #3060
-- python: remove arbitrary timeouts from tests #3059
-- refactorings #3026
-
-### Fixes
-- avoid archived, fresh chats #3053
-- Also resync UIDs in folders that are not configured #2289
-- treat "NO" IMAP response to MOVE and COPY commands as an error #3058
-- Fix a bug where messages in the Spam folder created contact requests #3015
-- Fix a bug where drafts disappeared after some days #3067
-- Parse MS Exchange read receipts and mark the original message as read #3075
-- do not retry message sending infinitely in case of permanent SMTP failure #3070
-- set message state to failed when retry limit is exceeded #3072
-
-
-## 1.75.0
-
-### Changes
-- optimize `delete_expired_imap_messages()` #3047
-
-
-## 1.74.0
-
-### Fixes
-- avoid reconnection loop when message without Message-ID is marked as seen #3044
-
-
-## 1.73.0
-
-### API changes
-- added `only_fetch_mvbox` config #3028
-
-### Changes
-- don't watch Sent folder by default #3025
-- use webxdc app name in chatlist/quotes/replies etc. #3027
-- make it possible to cancel message sending by removing the message #3034,
-  this was previosuly removed in 1.71.0 #2939
-- synchronize Seen flags only on watched folders to speed up
-  folder scanning #3041
-- remove direct dependency on `byteorder` crate #3031
-- refactorings #3023 #3013
-- update provider database #3043
-- improve documentation #3017 #3018 #3021
-
-### Fixes
-- fix splitting off text from webxdc messages #3032
-- call slow `delete_expired_imap_messages()` less often #3037
-- make synchronization of Seen status more robust in case unsolicited FETCH
-  result without UID is returned #3022
-- fetch Inbox before scanning folders to ensure iOS does
-  not kill the app before it gets to fetch the Inbox in background #3040
-
-
-## 1.72.0
-
-### Fixes
-- run migrations on backup import #3006
-
-
-## 1.71.0
-
-### API Changes
-- added APIs to handle database passwords: `dc_context_new_closed()`, `dc_context_open()`,
-  `dc_context_is_open()` and `dc_accounts_add_closed_account()` #2956 #2972
-- use second parameter of `dc_imex` to provide backup passphrase #2980
-- added `DC_MSG_WEBXDC`, `dc_send_webxdc_status_update()`,
-  `dc_get_webxdc_status_updates()`, `dc_msg_get_webxdc_blob()`, `dc_msg_get_webxdc_info()`
-  and `DC_EVENT_WEBXDC_STATUS_UPDATE` #2826 #2971 #2975 #2977 #2979 #2993 #2994 #2998 #3001 #3003
-- added `dc_msg_get_parent()` #2984
-- added `dc_msg_force_plaintext()` API for bots #2847
-- allow removing quotes on drafts `dc_msg_set_quote(msg, NULL)` #2950
-- removed `mvbox_watch` option; watching is enabled when `mvbox_move` is enabled #2906
-- removed `inbox_watch` option #2922
-- deprecated `os_name` in `dc_context_new()`, pass `NULL` or an empty string #2956
-
-### Changes
-- start making it possible to write to mailing lists #2736
-- add `hop_info` to `dc_get_info()` #2751 #2914 #2923
-- add information about whether the database is encrypted or not to `dc_get_info()` #3000
-- selfstatus now defaults to empty #2951 #2960
-- validate detached cryptographic signatures as used eg. by Thunderbird #2865
-- do not change the draft's `msg_id` on updates and sending #2887
-- add `imap` table to keep track of message UIDs #2909 #2938
-- replace `SendMsgToSmtp` jobs which stored outgoing messages in blobdir with `smtp` SQL table #2939 #2996
-- sql: enable `auto_vacuum=INCREMENTAL` #2931
-- sql: build rusqlite with sqlcipher #2934
-- synchronize Seen status across devices #2942
-- `dc_preconfigure_keypair` now takes ascii armored keys instead of base64 #2862
-- put removed member in Bcc instead of To in the message about removal #2864
-- improve group updates #2889
-- re-write the blob filename creation loop #2981
-- update provider database (11 Jan 2022) #2959
-- python: allow timeout for internal configure tracker API #2967
-- python: remove API deprecated in Python 3.10 #2907
-- refactorings #2932 #2957 #2947
-- improve tests #2863 #2866 #2881 #2908 #2918 #2901 #2973
-- improve documentation #2880 #2886 #2895
-- improve ci #2919 #2926 #2969 #2999
-
-### Fixes
-- fix leaving groups #2929
-- fix unread count #2861
-- make `add_parts()` not early-exit #2879
-- recognize MS Exchange read receipts as read receipts #2890
-- create parent directory if creating a new file fails #2978
-- save "configured" flag later #2974
-- improve log #2928
-- `dc_receive_imf`: don't fail on invalid address in the To field #2940
-
-
-## 1.70.0
-
-### Fixes
-- fix: do not abort Param parsing on unknown keys #2856
-- fix: execute `Chat-Group-Member-Removed:` even when arriving disordered #2857
-
-
-## 1.69.0
-
-### Fixes
-- fix group-related system messages in multi-device setups #2848
-- fix "Google Workspace" (former "G Suite") issues related to bad resolvers #2852
-
-
-## 1.68.0
-
-### Fixes
-- fix chat assignment when forwarding #2843
-- fix layout issues with the generated QR code svg #2842
-
-
-## 1.67.0
-
-### API changes
-- `dc_get_securejoin_qr_svg(chat_id)` added #2815
-- added stock-strings `DC_STR_SETUP_CONTACT_QR_DESC` and `DC_STR_SECURE_JOIN_GROUP_QR_DESC`
-
-
-## 1.66.0
-
-### API changes
-- `dc_contact_get_last_seen()` added #2823
-- python: `Contact.last_seen` added #2823
-- removed `DC_STR_NEWGROUPDRAFT`, we don't set draft after creating group anymore #2805
-
-### Changes
-- python: add cutil.from_optional_dc_charpointer() #2824
-- refactorings #2807 #2822 #2825
-
-
-## 1.65.0
-
-### Changes
-- python: add mypy support and some type hints #2809
-
-### Fixes
-- do not disable ephemeral timer when downloading a message partially #2811
-- apply existing ephemeral timer also to partially downloaded messages;
-  after full download, the ephemeral timer starts over #2811
-- replace user-visible error on verification failure with warning;
-  the error is logged to the corresponding chat anyway #2808
-
-
-## 1.64.0
-
-### Fixes
-- add 'waiting for being added to the group' only for group-joins,
-  not for setup-contact #2797
-- prioritize In-Reply-To: and References: headers over group IDs when assigning
-  messages to chats to fix incorrect assignment of Delta Chat replies to
-  classic email threads #2795
-
-
-## 1.63.0
-
-### API changes
-- `dc_get_last_error()` added #2788
-
-### Changes
-- Optimize Autocrypt gossip #2743
-
-### Fixes
-- fix permanently hiding of one-to-one chats after secure-join #2791
-
-
-## 1.62.0
-
-### API Changes
-- `dc_join_securejoin()` now always returns immediately;
-  the returned chat may not allow sending (`dc_chat_can_send()` returns false)
-  which may change as usual on `DC_EVENT_CHAT_MODIFIED` #2508 #2767
-- introduce multi-device-sync-messages;
-  as older cores display them as files in self-chat,
-  they are currently only sent if config option `send_sync_msgs` is set #2669
-- add `DC_EVENT_SELFAVATAR_CHANGED` #2742
-
-### Changes
-- use system DNS instead of google for MX queries #2780
-- improve error logging #2758
-- improve tests #2764 #2781
-- improve ci #2770
-- refactorings #2677 #2728 #2740 #2729 #2766 #2778
-
-### Fixes
-- add Let's Encrypt certificate to core as it may be missing older devices #2752
-- prioritize certificate setting from user over the one from provider-db #2749
-- fix "QR process failed" error #2725
-- do not update quota in endless loop #2726
-
-
-## 1.61.0
-
-### API Changes
-- download-on-demand added: `dc_msg_get_download_state()`, `dc_download_full_msg()`
-  and `download_limit` config option #2631 #2696
-- `dc_create_broadcast_list()` and chat type `DC_CHAT_TYPE_BROADCAST` added #2707 #2722
-- allow ui-specific configs using `ui.`-prefix in key (`dc_set_config(context, "ui.*", value)`) #2672
-- new strings from `DC_STR_PARTIAL_DOWNLOAD_MSG_BODY`
-  to `DC_STR_PART_OF_TOTAL_USED` #2631 #2694 #2707 #2723
-- emit warnings and errors from account manager with account-id 0 #2712
-
-### Changes
-- notify about incoming contact requests #2690
-- messages are marked as read on first read receipt #2699
-- quota warning reappears after import, rewarning at 95% #2702
-- lock strict TLS if certificate checks are automatic #2711
-- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
-- `Accounts` is not cloneable anymore #2654 #2658
-- update chat/contact data only when there was no newer update #2642
-- better detection of mailing list names #2665 #2685
-- log all decisions when applying ephemeral timer to chats #2679
-- connectivity view now translatable #2694 #2723
-- improve Doxygen documentation #2647 #2668 #2684 #2688 #2705
-- refactorings #2656 #2659 #2677 #2673 #2678 #2675 #2663 #2692 #2706
-- update provider database #2618
-
-### Fixes
-- ephemeral timer rollback protection #2693 #2709
-- recreate configured folders if they are deleted #2691
-- ignore MDNs sent to self #2674
-- recognize NDNs that put headers into "message/global-headers" part #2598
-- avoid `dc_get_contacts()` returning duplicate contact ids #2591
-- do not leak group names on forwarding messages #2719
-- in case of smtp-errors, iterate over all addresses to fix ipv6/v4 problems #2720
-- fix pkg-config file #2660
-- fix "QR process failed" error #2725
-
-
-## 1.60.0
-
-### Added
-- add device message to warn about QUOTA #2621
-- add SOCKS5 support #2474 #2620
-
-### Changes
-- don't emit multiple events with the same import/export progress number #2639
-- reduce message length limit to 5000 chars #2615
-
-### Fixes
-- keep event emitter from closing when there are no accounts #2636
-
-
-## 1.59.0
-
-### Added
-- add quota information to `dc_get_connectivity_html()`
-
-### Changes
-- refactorings #2592 #2570 #2581
-- add 'device chat about' to now existing status #2613
-- update provider database #2608
-
-### Fixes
-- provider database supports socket=PLAIN and dotless domains now #2604 #2608
-- add migrated accounts to events emitter #2607
-- fix forwarding quote-only mails #2600
-- do not set WantsMdn param for outgoing messages #2603
-- set timestamps for system messages #2593
-- do not treat gmail labels as folders #2587
-- avoid timing problems in `dc_maybe_network_lost()` #2551
-- only set smtp to "connected" if the last message was actually sent #2541
-
-
-## 1.58.0
-
-### Fixes
-- move WAL file together with database
-  and avoid using data if the database was not closed correctly before #2583
-
-
-## 1.57.0
-
-### API Changes
-
-- breaking change: removed deaddrop chat #2514 #2563
-
-  Contact request chats are not merged into a single virtual
-  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
-  same way as other chats, but sending of messages to them is not
-  allowed and MDNs are not sent automatically until the chat is
-  "accepted" by the user.
-
-  New API:
-  - `dc_chat_is_contact_request()`: returns true if chat is a contact
-    request.  In this case an option to accept the chat via
-    `dc_accept_chat()` should be shown in the UI.
-  - `dc_accept_chat()`: unblock the chat or accept contact request
-  - `dc_block_chat()`: block the chat, currently works only for mailing
-    lists.
-
-  Removed API:
-  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_decide_on_contact_request()`: this call requires a message ID
-    from deaddrop chat as input. As deaddrop chat is removed, this
-    call can't be used anymore.
-  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
-    only difference between these calls was in handling of deaddrop
-    chat
-  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
-
-- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
-  Instead, there is a new api `dc_get_connectivity()`
-  and `dc_get_connectivity_html()`;
-  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
-
-- breaking change: removed `dc_accounts_import_account()`
-  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
-  on its context
-
-- update account api, 2 new methods:
-  `int dc_all_work_done (dc_context_t* context);`
-  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
-
-- add api to check if a message was `Auto-Submitted`
-  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
-  python: `Message.is_bot()`
-
-- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
-  now returns `NULL` if there is no selected account
-
-- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
-  connectivity loss on its own (eg. iOS) #2550
-
-### Added
-- use Auto-Submitted: auto-generated header to identify bots #2502
-- allow sending stickers via repl tool
-- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
-- withdraw/revive own qr-codes #2512
-- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
-
-### Changes
-- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
-- update documentation #2548 #2561 #2569
-- breaking: `Accounts::create` does not also create an default account anymore #2500
-- remove "forwarded" from stickers, as the primary way of getting stickers
-  is by asking a bot and then forwarding them currently #2526
-- mimeparser: use mailparse to parse RFC 2231 filenames #2543
-- allow email addresses without dot in the domain part #2112
-- allow installing lib and include under different prefixes #2558
-- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
-- improve tests #2487 #2491 #2497
-- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
-- improve ci #2494
-- update provider-database #2565
-
-### Removed
-- remove `dc_accounts_import_account()` api #2521
-- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
-
-### Fixes
-- allow stickers with gif-images #2481
-- fix database migration #2486
-- do not count hidden messages in get_msg_cnt(). #2493
-- improve drafts detection #2489
-- fix panic when removing last, selected account from account manager #2500
-- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
-- avoid hiding outgoing classic emails #2505
-- fixes for message timestamps #2517
-- do not process names, avatars, location XMLs, message signature etc.
-  for duplicate messages #2513
-- fix `can_send` for users not in group #2479
-- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
-- fix which chats messages are assigned to #2465
-- fix: don't create chats when MDNs are received #2578
-
-
-## 1.56.0
-
-- fix downscaling images #2469
-
-- fix outgoing messages popping up in selfchat #2456
-
-- securejoin: display error reason if there is any #2470
-
-- do not allow deleting contacts with ongoing chats #2458
-
-- fix: ignore drafts folder when scanning #2454
-
-- fix: scan folders also when inbox is not watched #2446
-
-- more robust In-Reply-To parsing #2182
-
-- update dependencies #2441 #2438 #2439 #2440 #2447 #2448 #2449 #2452 #2453 #2460 #2464 #2466
-
-- update provider-database #2471
-
-- refactorings #2459 #2457
-
-- improve tests and ci #2445 #2450 #2451
-
-
-## 1.55.0
-
-- fix panic when receiving some HTML messages #2434
-
-- fix downloading some messages multiple times #2430
-
-- fix formatting of read receipt texts #2431
-
-- simplify SQL error handling #2415
-
-- explicit rust API for creating chats with blocked status #2282
-
-- debloat the binary by using less AsRef arguments #2425
-
-
-## 1.54.0
-
-- switch back from `sqlx` to `rusqlite` due to performance regressions #2380 #2381 #2385 #2387
-
-- global search performance improvement #2364 #2365 #2366
-
-- improve SQLite performance with `PRAGMA synchronous=normal` #2382
-
-- python: fix building of bindings against system-wide install of `libdeltachat` #2383 #2385
-
-- python: list `requests` as a requirement #2390
-
-- fix creation of many delete jobs when being offline #2372
-
-- synchronize status between devices #2386
-
-- deaddrop (contact requests) chat improvements #2373
-
-- add "Forwarded:" to notification and chatlist summaries #2310
-
-- place user avatar directly into `Chat-User-Avatar` header #2232 #2384
-
-- improve tests #2360 #2362 #2370 #2377 #2387
-
-- cleanup #2359 #2361 #2374 #2376 #2379 #2388
-
+# Changelog 
 
 ## 1.53.0
 

CMakeLists.txt

@@ -1,38 +1,15 @@
 cmake_minimum_required(VERSION 3.16)
-project(deltachat LANGUAGES C)
-include(GNUInstallDirs)
+project(deltachat)
 
 find_program(CARGO cargo)
 
-if(APPLE)
-       set(DYNAMIC_EXT "dylib")
-elseif(UNIX)
-       set(DYNAMIC_EXT "so")
-else()
-       set(DYNAMIC_EXT "dll")
-endif()
-
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
-	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --package deltachat_ffi --release
+	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
 add_custom_target(
@@ -40,11 +17,12 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
+	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
+include(GNUInstallDirs)
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,10 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.79.0"
+version = "1.53.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2021"
+edition = "2018"
 license = "MPL-2.0"
-rust-version = "1.56"
 
 [profile.dev]
 debug = 0
@@ -13,83 +12,80 @@ debug = 0
 lto = true
 
 [dependencies]
-deltachat_derive = { path = "./deltachat_derive" }
-
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", default-features=false, features = ["smtp-transport", "socks5"] }
-async-std-resolver = "0.21"
-async-std = { version = "1" }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
-backtrace = "0.3"
+anyhow = "1.0.28"
+async-imap = "0.4.0"
+async-native-tls = { version = "0.3.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
+async-std-resolver = "0.19.5"
+async-std = { version = "~1.8.0", features = ["unstable"] }
+async-tar = "0.3.0"
+async-trait = "0.1.31"
+backtrace = "0.3.33"
 base64 = "0.13"
-bitflags = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
+bitflags = "1.1.0"
+byteorder = "1.3.1"
+charset = "0.1"
+chrono = "0.4.6"
+dirs = { version = "3.0.1", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+escaper = "0.1.0"
+futures = "0.3.4"
 hex = "0.4.0"
-image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+indexmap = "1.3.0"
+itertools = "0.10.0"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
-log = {version = "0.4.16", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
-once_cell = "1.10.0"
+libc = "0.2.51"
+log = {version = "0.4.8", optional = true }
+mailparse = "0.13.0"
+native-tls = "0.2.3"
+num_cpus = "1.13.0"
+num-derive = "0.3.0"
+num-traits = "0.2.6"
+once_cell = "1.4.1"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.20"
-rand = "0.7"
-regex = "1.5"
-rusqlite = { version = "0.27", features = ["sqlcipher"] }
-rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
+pgp = { version = "0.7.0", default-features = false }
+pretty_env_logger = { version = "0.4.0", optional = true }
+quick-xml = "0.18.1"
+rand = "0.7.0"
+regex = "1.1.6"
+rust-hsluv = "0.1.4"
+rustyline = { version = "4.1.0", optional = true }
+sanitize-filename = "0.3.0"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.10"
-sha2 = "0.10"
-smallvec = "1"
-strum = "0.24"
-strum_macros = "0.24"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
+sha-1 = "0.9.3"
+sha2 = "0.9.0"
+smallvec = "1.0.0"
+sqlx = { git = "https://github.com/deltachat/sqlx", branch = "master", features = ["runtime-async-std-native-tls", "sqlite"] }
+# keep in sync with sqlx
+libsqlite3-sys = { version = "0.22.0", default-features = false, features = [ "pkg-config", "vcpkg", "bundled" ] }
+stop-token = { version = "0.1.1", features = ["unstable"] }
+strum = "0.20.0"
+strum_macros = "0.20.1"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+thiserror = "1.0.14"
+toml = "0.5.6"
+url = "2.1.1"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "4.3.3"
-textwrap = "0.15.0"
-zip = { version = "0.6.2", default-features = false, features = ["deflate"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = { version = "0.3.4", features = ["async_std"] }
-futures-lite = "1.12"
-log = "0.4"
-pretty_env_logger = "0.4"
-proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+criterion = "0.3"
+futures-lite = "1.7.0"
+log = "0.4.11"
+pretty_assertions = "0.6.1"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+tempfile = "3.0"
 
 [workspace]
 members = [
   "deltachat-ffi",
-  "deltachat_derive",
 ]
 
 [[example]]
@@ -115,21 +111,9 @@ harness = false
 name = "search_msgs"
 harness = false
 
-[[bench]]
-name = "receive_emails"
-harness = false
-
-[[bench]]
-name = "get_chat_msgs"
-harness = false
-
-[[bench]]
-name = "get_chatlist"
-harness = false
-
 [features]
-default = ["vendored"]
+default = []
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
+vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]

README.md

@@ -3,6 +3,8 @@
 > Deltachat-core written in Rust 
 
 [![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![Remote tests](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/remote_tests.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -12,8 +14,6 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
-> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
-
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:
@@ -81,16 +81,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh
@@ -127,11 +117,11 @@ $ cargo test -- --ignored
 
 Language bindings are available for:
 
-- **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
-- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
-- **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/deltachat/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library

assets/icon-broadcast.svg

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:xlink="http://www.w3.org/1999/xlink"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   enable-background="new 0 0 128 128"
-   viewBox="0 0 60 60"
-   version="1.1"
-   id="svg878"
-   sodipodi:docname="icon-broadcast.svg"
-   width="60"
-   height="60"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-broadcast.png"
-   inkscape:export-xdpi="409.60001"
-   inkscape:export-ydpi="409.60001">
-  <metadata
-     id="metadata884">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs882" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1329"
-     inkscape:window-height="847"
-     id="namedview880"
-     showgrid="false"
-     inkscape:zoom="5.21875"
-     inkscape:cx="36.598802"
-     inkscape:cy="32.191617"
-     inkscape:window-x="111"
-     inkscape:window-y="205"
-     inkscape:window-maximized="0"
-     inkscape:current-layer="svg878"
-     inkscape:document-rotation="0" />
-  <radialGradient
-     id="c"
-     cx="65.25"
-     cy="89"
-     r="26.440001"
-     gradientTransform="matrix(0.77611266,0.11996647,-0.18999676,1.2286617,-11.305867,-60.065999)"
-     gradientUnits="userSpaceOnUse">
-    <stop
-       stop-color="#FFC107"
-       offset="0"
-       id="stop833" />
-    <stop
-       stop-color="#FFBD06"
-       offset=".3502"
-       id="stop835" />
-    <stop
-       stop-color="#FFB104"
-       offset=".6938"
-       id="stop837" />
-    <stop
-       stop-color="#FFA000"
-       offset="1"
-       id="stop839" />
-  </radialGradient>
-  <radialGradient
-     id="b"
-     cx="52.5"
-     cy="19.75"
-     r="92.975998"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)">
-    <stop
-       stop-color="#EF5350"
-       offset="0"
-       id="stop848" />
-    <stop
-       stop-color="#EB4F4C"
-       offset=".246"
-       id="stop850" />
-    <stop
-       stop-color="#E04341"
-       offset=".4878"
-       id="stop852" />
-    <stop
-       stop-color="#CD302F"
-       offset=".7272"
-       id="stop854" />
-    <stop
-       stop-color="#C62828"
-       offset=".8004"
-       id="stop856" />
-    <stop
-       stop-color="#C62828"
-       offset="1"
-       id="stop858" />
-  </radialGradient>
-  <radialGradient
-     id="a"
-     cx="16.979"
-     cy="92"
-     r="24.165001"
-     gradientUnits="userSpaceOnUse"
-     gradientTransform="rotate(45.323856,68.997115,75.979538)"
-     xlink:href="#b">
-    <stop
-       stop-color="#E0E0E0"
-       offset="0"
-       id="stop863" />
-    <stop
-       stop-color="#CFCFCF"
-       offset=".3112"
-       id="stop865" />
-    <stop
-       stop-color="#A4A4A4"
-       offset=".9228"
-       id="stop867" />
-    <stop
-       stop-color="#9E9E9E"
-       offset="1"
-       id="stop869" />
-  </radialGradient>
-  <rect
-     y="0"
-     x="0"
-     height="60"
-     width="60"
-     id="rect1420"
-     style="fill:#7cc0bc;fill-opacity:1;stroke:none;stroke-width:1.29077" />
-  <path
-     id="path872"
-     style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.336872;stroke-opacity:1"
-     d="m 8.6780027,35.573064 0.032831,-11.910176 c 0.00138,-0.476406 0.4881282,-0.794259 0.9235226,-0.604877 l 4.1144877,2.345752 -0.02386,8.656315 -4.1268029,2.122946 C 9.1617452,36.370003 8.6766889,36.049472 8.6780027,35.573064 Z m 5.0469633,-1.508222 0.02386,-8.656314 31.145424,-9.537653 c 0.841472,-0.219211 1.65915,0.41667 1.656755,1.283728 l -0.06929,25.139995 c -0.0024,0.867062 -0.825942,1.500799 -1.663803,1.274581 z m 3.8042,6.892234 C 16.681121,40.104348 16.315444,38.819414 16.69043,37.591308 l 2.252234,-7.347193 c 0.2644,-0.861571 0.845185,-1.567441 1.641953,-1.989251 0.796769,-0.421808 1.706956,-0.509819 2.568531,-0.245419 l 7.263888,2.225804 c 1.775518,0.543235 2.780299,2.432591 2.232297,4.208094 L 30.3971,41.790532 c -0.545627,1.777887 -2.432591,2.780297 -4.208095,2.232298 l -7.263891,-2.225804 c -0.545033,-0.165864 -1.01825,-0.460162 -1.395948,-0.83995 z m 12.377693,-7.976728 c -0.07601,-0.07642 -0.17114,-0.133864 -0.280621,-0.167516 l -7.263891,-2.225803 c -0.233244,-0.07209 -0.421626,0.0013 -0.512275,0.04861 -0.09064,0.0474 -0.25772,0.166033 -0.327435,0.396899 l -2.252234,7.347191 c -0.108166,0.354628 0.09088,0.731541 0.447888,0.842099 l 7.263891,2.225802 c 0.354626,0.108174 0.731539,-0.09088 0.842099,-0.447888 l 2.249845,-7.344814 c 0.07453,-0.245145 0.0014,-0.504991 -0.167267,-0.67458 z" />
-</svg>

assets/icon-webxdc.svg

@@ -1,181 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   width="80mm"
-   height="297mm"
-   viewBox="0 0 80 297"
-   version="1.1"
-   id="svg71"
-   inkscape:version="1.0.2 (e86c8708, 2021-01-15)"
-   sodipodi:docname="icon-webxdc.svg"
-   inkscape:export-filename="C:\Users\user\OneDrive - BFW-Leipzig\Documents\LogoDC\finalohnerand.png"
-   inkscape:export-xdpi="96"
-   inkscape:export-ydpi="96">
-  <metadata
-     id="metadata856">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-        <dc:title></dc:title>
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <sodipodi:namedview
-     id="namedview73"
-     pagecolor="#767676"
-     bordercolor="#666666"
-     borderopacity="1.0"
-     inkscape:pageshadow="2"
-     inkscape:pageopacity="0"
-     inkscape:pagecheckerboard="true"
-     inkscape:document-units="mm"
-     showgrid="false"
-     showborder="false"
-     inkscape:snap-bbox="true"
-     inkscape:bbox-paths="true"
-     inkscape:bbox-nodes="true"
-     inkscape:snap-bbox-edge-midpoints="true"
-     inkscape:snap-bbox-midpoints="true"
-     inkscape:object-paths="true"
-     inkscape:snap-intersection-paths="true"
-     inkscape:zoom="1.4142136"
-     inkscape:cx="-90.271136"
-     inkscape:cy="-1233.1209"
-     inkscape:window-width="1864"
-     inkscape:window-height="1027"
-     inkscape:window-x="56"
-     inkscape:window-y="25"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="layer1"
-     inkscape:snap-global="false"
-     showguides="false"
-     inkscape:guide-bbox="true"
-     inkscape:document-rotation="0"
-     units="px">
-    <sodipodi:guide
-       position="-154.76097,641.11689"
-       orientation="0,-1"
-       id="guide21118" />
-    <sodipodi:guide
-       position="-60.286487,633.36619"
-       orientation="0,-1"
-       id="guide21120" />
-  </sodipodi:namedview>
-  <defs
-     id="defs68">
-    <linearGradient
-       id="linearGradient4375">
-      <stop
-         style="stop-color:#364e59;stop-opacity:1;"
-         offset="0"
-         id="stop4377" />
-      <stop
-         style="stop-color:#364e59;stop-opacity:0;"
-         offset="1"
-         id="stop4379" />
-    </linearGradient>
-  </defs>
-  <g
-     inkscape:label="Ebene 1"
-     inkscape:groupmode="layer"
-     id="layer1">
-    <rect
-       style="fill:#1a1a1a;stroke:#000000;stroke-width:0.167903"
-       id="rect880"
-       width="79.8321"
-       height="79.8321"
-       x="-64.03286"
-       y="-375.9097"
-       ry="0" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path3799-2"
-       d="m -24.089585,-372.59579 c -19.986026,0.24336 -36.196903,16.666 -36.196903,36.67011 0,20.00409 16.210877,36.03233 36.196903,35.78912 19.0024236,-0.076 14.5340713,-10.6146 35.538854,-0.85693 -11.50627538,-17.97454 0.390097,-20.36737 0.658079,-35.81316 0,-20.00411 -16.2108788,-36.03235 -36.196911,-35.78914 z"
-       style="fill:#364e59;fill-opacity:1;stroke:none;stroke-width:1.93355"
-       sodipodi:nodetypes="sscccs" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -54.193871,-325.26419 Z"
-       id="path3846" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.397951,-326.67773 Z"
-       id="path3848" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="m -49.397951,-326.67773 v 0 0"
-       id="path3850" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.01;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"
-       d="m -51.35133,-325.0334 -7.964067,5.98895 z"
-       id="path3965" />
-    <path
-       inkscape:connector-curvature="0"
-       id="path11037"
-       d="m -24.089585,-372.19891 c -19.986026,0.24156 -36.196903,16.54352 -36.196903,36.40062 0,7.86524 2.543315,15.1113 6.857155,20.97971 6.577146,8.94734 11.123515,9.77363 11.123515,9.77363 1.343237,1.78324 10.270932,4.3223 10.270932,4.3223 l 16.791727,-70.86654 -0.468369,-0.33457 c 0.458597,0.26445 0.428277,-0.27515 -8.378035,-0.27515 z"
-       style="fill:#7cc5cc;fill-opacity:1;stroke:none;stroke-width:1.92643"
-       sodipodi:nodetypes="sssccccss" />
-    <path
-       style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1"
-       d="M -49.944239,-310.69957 Z"
-       id="path13674" />
-    <g
-       id="g15178"
-       transform="matrix(0.79975737,0,0,0.79975737,53.088959,-63.716396)">
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072"
-         width="29.897917"
-         height="6.8791666"
-         x="-334.4964"
-         y="-154.51025"
-         transform="rotate(45)" />
-      <rect
-         style="fill:#364e59;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-         id="rect15072-5"
-         width="29.897917"
-         height="6.8791666"
-         x="147.63107"
-         y="-334.4964"
-         transform="rotate(-45)"
-         inkscape:transform-center-x="-0.74835017"
-         inkscape:transform-center-y="0.37417525" />
-    </g>
-    <g
-       id="g22468"
-       transform="translate(3.3033974)">
-      <g
-         id="g15178-0"
-         transform="matrix(-0.79975737,0,0,0.79975737,-103.11028,-63.716404)"
-         style="fill:#7cc5cc;fill-opacity:1">
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-2"
-           width="29.897917"
-           height="6.8791666"
-           x="-334.4964"
-           y="-154.51025"
-           transform="rotate(45)" />
-        <rect
-           style="fill:#7cc5cc;fill-opacity:1;stroke-width:0.01;stroke-miterlimit:4;stroke-dasharray:none"
-           id="rect15072-5-5"
-           width="29.897917"
-           height="6.8791666"
-           x="147.63107"
-           y="-334.4964"
-           transform="rotate(-45)"
-           inkscape:transform-center-x="-0.74835017"
-           inkscape:transform-center-y="0.37417525" />
-      </g>
-    </g>
-  </g>
-</svg>

assets/qrcode_logo_footer.svg

@@ -1,10 +0,0 @@
-<text
-     xml:space="preserve"
-     style="font-weight:bold;font-size:24.4118px;line-height:1.25;font-family:sans-serif;fill:#aaaaaa;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     x="42.325161"
-     y="23.32255"
-     id="text72398">get.delta.chat</text>
-  <path
-     id="path84310"
-     style="opacity:0.25;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.915439"
-     d="M 17.13769,0.00129321 C 7.6753075,0.11650893 0,7.8915283 0,17.362467 c 0,9.47094 7.6753075,17.059745 17.13769,16.944599 8.99669,-0.03598 6.880074,-5.025654 16.824785,-0.405885 -5.447648,-8.510047 0.184241,-9.642482 0.311117,-16.955289 0,-9.4709395 -7.673512,-17.0597453 -17.135895,-16.94459879 z M 17.0769,4.9986797 c 1.84214,0 3.447355,0.253959 4.815003,0.7616693 1.381603,0.5076411 2.072253,1.207862 2.072253,2.0990711 0,0.4286855 -0.167495,0.7836052 -0.50242,1.0656242 -0.334921,0.2819844 -0.724544,0.4237724 -1.171121,0.4237724 -0.641952,0 -1.396532,-0.3909376 -2.261778,-1.169353 C 19.14963,7.3898036 18.402555,6.83791 17.788507,6.5220182 17.188416,6.1950547 16.484552,6.0321266 15.675129,6.0321266 c -1.032717,0 -1.883352,0.1854523 -2.553215,0.5578447 -0.655913,0.372254 -0.98517,0.8460916 -0.98517,1.4214436 0,0.5414792 0.272815,1.0495355 0.817093,1.5233385 0.544275,0.4738026 1.946291,1.3367446 4.207097,2.5889976 2.414319,1.342419 4.117377,2.390985 5.108232,3.146807 1.004795,0.755857 1.821505,1.675853 2.449514,2.758846 0.628002,1.082993 0.942253,2.227607 0.942253,3.434674 0,2.120834 -0.929555,3.993314 -2.785656,5.617786 -1.84214,1.613228 -3.99694,2.41915 -6.467082,2.41915 -2.246845,0 -4.145607,-0.647976 -5.694677,-1.945312 -1.5490699,-1.297336 -2.3225722,-3.028063 -2.3225722,-5.194049 0,-2.087031 0.8506345,-3.83094 2.5532182,-5.229825 1.716541,-1.398884 3.824203,-2.245599 6.322256,-2.538897 -0.697774,-0.631749 -1.668763,-1.387225 -2.910816,-2.267155 -1.367648,-0.970199 -2.287914,-1.73045 -2.762402,-2.283243 -0.474491,-0.5640381 -0.711618,-1.1795944 -0.711618,-1.8451814 0,-0.9927581 0.572093,-1.7710351 1.716451,-2.3351077 1.144362,-0.5753173 2.636724,-0.8635642 4.478865,-0.8635642 z m 1.110327,10.3738083 c -4.005262,0.5302 -6.007576,2.75279 -6.007576,6.667322 0,2.01932 0.49495,3.587291 1.485805,4.704157 1.004806,1.116832 2.169696,1.675299 3.495479,1.675299 1.381602,0 2.520072,-0.535632 3.413229,-1.60738 0.893168,-1.082959 1.339187,-2.545264 1.339187,-4.384079 0,-2.662348 -1.242022,-5.013441 -3.726124,-7.055319 z" />

benches/contacts.rs

@@ -8,14 +8,16 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
         .collect::<Vec<String>>()
         .join("");
 
-    Contact::add_address_book(&context, &book).await.unwrap();
+    Contact::add_address_book(&context, book).await.unwrap();
 
     let query: Option<&str> = None;
     for _ in 0..read_count {

benches/create_account.rs

@@ -8,7 +8,7 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts").into();
 
-    let mut accounts = Accounts::new(p.clone()).await.unwrap();
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

benches/get_chat_msgs.rs

@@ -1,40 +0,0 @@
-use async_std::path::Path;
-
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chat::{self, ChatId};
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    for c in chats.iter().take(10) {
-        black_box(chat::get_chat_msgs(&context, *c, 0, None).await.ok());
-    }
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
-            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
-            let len = chatlist.len();
-            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
-        });
-
-        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/get_chatlist.rs

@@ -1,27 +0,0 @@
-use criterion::async_executor::AsyncStdExecutor;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
-use deltachat::chatlist::Chatlist;
-use deltachat::context::Context;
-
-async fn get_chat_list_benchmark(context: &Context) {
-    Chatlist::try_load(context, 0, None, None).await.unwrap();
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
-    // messages, such as your primary account.
-    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
-        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
-            b.to_async(AsyncStdExecutor)
-                .iter(|| get_chat_list_benchmark(black_box(&context)))
-        });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
-    }
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/receive_emails.rs

@@ -1,84 +0,0 @@
-use async_std::{path::PathBuf, task::block_on};
-use criterion::{
-    async_executor::AsyncStdExecutor, black_box, criterion_group, criterion_main, BatchSize,
-    Criterion,
-};
-use deltachat::{
-    config::Config,
-    context::Context,
-    dc_receive_imf::dc_receive_imf,
-    imex::{imex, ImexMode},
-};
-use tempfile::tempdir;
-
-async fn recv_all_emails(context: Context) -> Context {
-    for i in 0..100 {
-        let imf_raw = format!(
-            "Subject: Benchmark
-Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
-Date: Sat, 07 Dec 2019 19:00:27 +0000
-To: alice@example.com
-From: sender@testrun.org
-Chat-Version: 1.0
-Chat-Disposition-Notification-To: sender@testrun.org
-Chat-User-Avatar: 0
-In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
-MIME-Version: 1.0
-
-Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
-
-Hello {i}",
-            i = i,
-            i_dec = i - 1,
-        );
-        dc_receive_imf(&context, black_box(imf_raw.as_bytes()), false)
-            .await
-            .unwrap();
-    }
-    context
-}
-
-async fn create_context() -> Context {
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
-
-    let backup: PathBuf = std::env::current_dir()
-        .unwrap()
-        .join("delta-chat-backup.tar")
-        .into();
-    if backup.exists().await {
-        println!("Importing backup");
-        imex(&context, ImexMode::ImportBackup, &backup, None)
-            .await
-            .unwrap();
-    }
-
-    let addr = "alice@example.com";
-    context.set_config(Config::Addr, Some(addr)).await.unwrap();
-    context
-        .set_config(Config::ConfiguredAddr, Some(addr))
-        .await
-        .unwrap();
-    context
-        .set_config(Config::Configured, Some("1"))
-        .await
-        .unwrap();
-    context
-}
-
-fn criterion_benchmark(c: &mut Criterion) {
-    let mut group = c.benchmark_group("Receive messages");
-    group.bench_function("Receive 100 simple text msgs", |b| {
-        b.to_async(AsyncStdExecutor).iter_batched(
-            || block_on(create_context()),
-            |context| recv_all_emails(black_box(context)),
-            BatchSize::LargeInput,
-        );
-    });
-    group.finish();
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);

benches/search_msgs.rs

@@ -6,7 +6,9 @@ use std::path::Path;
 async fn search_benchmark(path: impl AsRef<Path>) {
     let dbfile = path.as_ref();
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new("FakeOS".into(), dbfile.into(), id)
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();
@@ -20,8 +22,6 @@ fn criterion_benchmark(c: &mut Criterion) {
         c.bench_function("search hello", |b| {
             b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
         });
-    } else {
-        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.79.0"
+version = "1.53.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -17,13 +17,13 @@ crate-type = ["cdylib", "staticlib"]
 [dependencies]
 deltachat = { path = "../", default-features = false }
 libc = "0.2"
-human-panic = "1"
-num-traits = "0.2"
+human-panic = "1.0.1"
+num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1"
-anyhow = "1"
-thiserror = "1"
-rand = "0.7"
+async-std = "1.6.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -583,7 +583,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,16 +4,4 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
-	border-radius: 6px;
 }
-
-code {
-	background-color: #e0e0e0;
-	padding-left: .5em;
-	padding-right: .5em;
-	border-radius: 6px;
-}
-
-li {
-	margin-bottom: .5em;
-}

deltachat-ffi/README.md

@@ -1,9 +1,5 @@
 # Delta Chat C Interface
 
-## Installation
-
-see `Installing libdeltachat system wide` in [../README.md](../README.md)
-
 ## Documentation
 
 To generate the C Interface documentation,

deltachat-ffi/build.rs

@@ -23,13 +23,11 @@ fn main() {
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
         prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/lot.rs

@@ -1,245 +0,0 @@
-//! # Legacy generic return values for C API.
-
-use crate::message::MessageState;
-use crate::qr::Qr;
-use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
-
-/// An object containing a set of values.
-/// The meaning of the values is defined by the function returning the object.
-/// Lot objects are created
-/// eg. by chatlist.get_summary() or dc_msg_get_summary().
-///
-/// *Lot* is used in the meaning *heap* here.
-#[derive(Debug)]
-pub enum Lot {
-    Summary(Summary),
-    Qr(Qr),
-    Error(String),
-}
-
-#[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Meaning {
-    None = 0,
-    Text1Draft = 1,
-    Text1Username = 2,
-    Text1Self = 3,
-}
-
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
-impl Lot {
-    pub fn get_text1(&self) -> Option<&str> {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => None,
-                Some(SummaryPrefix::Draft(text)) => Some(text),
-                Some(SummaryPrefix::Username(username)) => Some(username),
-                Some(SummaryPrefix::Me(text)) => Some(text),
-            },
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => None,
-                Qr::AskVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::FprOk { .. } => None,
-                Qr::FprMismatch { .. } => None,
-                Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
-                Qr::Account { domain } => Some(domain),
-                Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
-                Qr::Url { url } => Some(url),
-                Qr::Text { text } => Some(text),
-                Qr::WithdrawVerifyContact { .. } => None,
-                Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
-                Qr::ReviveVerifyContact { .. } => None,
-                Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
-            },
-            Self::Error(err) => Some(err),
-        }
-    }
-
-    pub fn get_text2(&self) -> Option<Cow<str>> {
-        match self {
-            Self::Summary(summary) => Some(summary.truncated_text(160)),
-            Self::Qr(_) => None,
-            Self::Error(_) => None,
-        }
-    }
-
-    pub fn get_text1_meaning(&self) -> Meaning {
-        match self {
-            Self::Summary(summary) => match &summary.prefix {
-                None => Meaning::None,
-                Some(SummaryPrefix::Draft(_text)) => Meaning::Text1Draft,
-                Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
-                Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
-            },
-            Self::Qr(_qr) => Meaning::None,
-            Self::Error(_err) => Meaning::None,
-        }
-    }
-
-    pub fn get_state(&self) -> LotState {
-        match self {
-            Self::Summary(summary) => summary.state.into(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { .. } => LotState::QrAskVerifyContact,
-                Qr::AskVerifyGroup { .. } => LotState::QrAskVerifyGroup,
-                Qr::FprOk { .. } => LotState::QrFprOk,
-                Qr::FprMismatch { .. } => LotState::QrFprMismatch,
-                Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
-                Qr::Account { .. } => LotState::QrAccount,
-                Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
-                Qr::Addr { .. } => LotState::QrAddr,
-                Qr::Url { .. } => LotState::QrUrl,
-                Qr::Text { .. } => LotState::QrText,
-                Qr::WithdrawVerifyContact { .. } => LotState::QrWithdrawVerifyContact,
-                Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
-                Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
-                Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
-            },
-            Self::Error(_err) => LotState::QrError,
-        }
-    }
-
-    pub fn get_id(&self) -> u32 {
-        match self {
-            Self::Summary(_) => Default::default(),
-            Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => contact_id.to_u32(),
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
-                Qr::FprWithoutAddr { .. } => Default::default(),
-                Qr::Account { .. } => Default::default(),
-                Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
-                Qr::Url { .. } => Default::default(),
-                Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
-                Qr::ReviveVerifyGroup { .. } => Default::default(),
-            },
-            Self::Error(_) => Default::default(),
-        }
-    }
-
-    pub fn get_timestamp(&self) -> i64 {
-        match self {
-            Self::Summary(summary) => summary.timestamp,
-            Self::Qr(_) => Default::default(),
-            Self::Error(_) => Default::default(),
-        }
-    }
-}
-
-#[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum LotState {
-    // Default
-    Undefined = 0,
-
-    // Qr States
-    /// id=contact
-    QrAskVerifyContact = 200,
-
-    /// text1=groupname
-    QrAskVerifyGroup = 202,
-
-    /// id=contact
-    QrFprOk = 210,
-
-    /// id=contact
-    QrFprMismatch = 220,
-
-    /// text1=formatted fingerprint
-    QrFprWithoutAddr = 230,
-
-    /// text1=domain
-    QrAccount = 250,
-
-    /// text1=domain, text2=instance pattern
-    QrWebrtcInstance = 260,
-
-    /// id=contact
-    QrAddr = 320,
-
-    /// text1=text
-    QrText = 330,
-
-    /// text1=URL
-    QrUrl = 332,
-
-    /// text1=error string
-    QrError = 400,
-
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
-    // Message States
-    MsgInFresh = 10,
-    MsgInNoticed = 13,
-    MsgInSeen = 16,
-    MsgOutPreparing = 18,
-    MsgOutDraft = 19,
-    MsgOutPending = 20,
-    MsgOutFailed = 24,
-    MsgOutDelivered = 26,
-    MsgOutMdnRcvd = 28,
-}
-
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        Lot::Summary(summary)
-    }
-}
-
-impl From<Qr> for Lot {
-    fn from(qr: Qr) -> Self {
-        Lot::Qr(qr)
-    }
-}
-
-// Make it easy to convert errors into the final `Lot`.
-impl From<Error> for Lot {
-    fn from(error: Error) -> Self {
-        Lot::Error(error.to_string())
-    }
-}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -167,20 +170,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +186,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name

deltachat_derive/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "deltachat_derive"
-version = "2.0.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2018"
-license = "MPL-2.0"
-
-[lib]
-proc-macro = true
-
-[dependencies]
-syn = "1"
-quote = "1"

deltachat_derive/src/lib.rs

@@ -1,47 +0,0 @@
-#![recursion_limit = "128"]
-extern crate proc_macro;
-
-use crate::proc_macro::TokenStream;
-use quote::quote;
-
-// For now, assume (not check) that these macroses are applied to enum without
-// data.  If this assumption is violated, compiler error will point to
-// generated code, which is not very user-friendly.
-
-#[proc_macro_derive(ToSql)]
-pub fn to_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::ToSql for #name {
-            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-                let num = *self as i64;
-                let value = rusqlite::types::Value::Integer(num);
-                let output = rusqlite::types::ToSqlOutput::Owned(value);
-                std::result::Result::Ok(output)
-            }
-        }
-    };
-    gen.into()
-}
-
-#[proc_macro_derive(FromSql)]
-pub fn from_sql_derive(input: TokenStream) -> TokenStream {
-    let ast: syn::DeriveInput = syn::parse(input).unwrap();
-    let name = &ast.ident;
-
-    let gen = quote! {
-        impl rusqlite::types::FromSql for #name {
-            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-                let inner = rusqlite::types::FromSql::column_result(col)?;
-                if let Some(value) = num_traits::FromPrimitive::from_i64(inner) {
-                    Ok(value)
-                } else {
-                    Err(rusqlite::types::FromSqlError::OutOfRange(inner))
-                }
-            }
-        }
-    };
-    gen.into()
-}

draft/aeap-mvp.rst

@@ -1,33 +0,0 @@
-AEAP MVP
-========
-
-Changes to the UIs
-------------------
-
-- The secondary self addresses (see below) are shown in the UI, but not editable.
-
-- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
-
-Changes in the core
--------------------
-
-- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
-
-- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
-
-  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
-
-  - The key stays the same.
-
-- No changes for 1:1 chats, there simply is a new one
-
-- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
-  
-  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
-
-  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
-
-Other
------
-
-- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/webxdc-dev-reference.md

@@ -1,185 +0,0 @@
-# Webxdc Developer Reference
-
-## Webxdc File Format
-
-- a **Webxdc app** is a **ZIP-file** with the extension `.xdc`
-- the ZIP-file must use the default compression methods as of RFC 1950,
-  this is "Deflate" or "Store"
-- the ZIP-file must contain at least the file `index.html`
-- if the Webxdc app is started, `index.html` is opened in a restricted webview
-  that allow accessing resources only from the ZIP-file
-
-
-## Webxdc API
-
-There are some additional APIs available once `webxdc.js` is included
-(the file will be provided by the concrete implementations,
-no need to add `webxdc.js` to your ZIP-file):
-
-```html
-<script src="webxdc.js"></script>
-```
-
-### sendUpdate()
-
-```js
-window.webxdc.sendUpdate(update, descr);
-```
-
-Webxdc apps are usually shared in a chat and run independently on each peer.
-To get a shared state, the peers use `sendUpdate()` to send updates to each other.
-
-- `update`: an object with the following properties:  
-    - `update.payload`: any javascript primitive, array or object.
-    - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
-       use this option sparingly to not spam the chat.
-    - `update.summary`: optional, short text, shown beside app icon;
-       it is recommended to use some aggregated value,  eg. "8 votes", "Highscore: 123"
-
-- `descr`: short, human-readable description what this update is about.
-  this is shown eg. as a fallback text in an email program.
-
-All peers, including the sending one,
-will receive the update by the callback given to `setUpdateListener()`.
-
-There are situations where the user cannot send messages to a chat,
-eg. if the webxdc instance comes as a contact request or if the user has left a group.
-In these cases, you can still call `sendUpdate()`,
-however, the update won't be sent to other peers
-and you won't get the update by `setUpdateListener()`.
-
-
-### setUpdateListener()
-
-```js
-let promise = window.webxdc.setUpdateListener((update) => {}, serial);
-```
-
-With `setUpdateListener()` you define a callback that receives the updates
-sent by `sendUpdate()`. The callback is called for updates sent by you or other peers.
-The `serial` specifies the last serial that you know about (defaults to 0). 
-The returned promise resolves when the listener has processed all the update messages known at the time when  `setUpdateListener` was called. 
-
-Each `update` which is passed to the callback comes with the following properties: 
-
-- `update.payload`: equals the payload given to `sendUpdate()`
-
-- `update.serial`: the serial number of this update.
-  Serials are larger `0` and newer serials have higher numbers.
-  There may be gaps in the serials
-  and it is not guaranteed that the next serial is exactly incremented by one.
-
-- `update.max_serial`: the maximum serial currently known.
-  If `max_serial` equals `serial` this update is the last update (until new network messages arrive).
-
-- `update.info`: optional, short, informational message (see `send_update`) 
-
-- `update.summary`: optional, short text, shown beside app icon (see `send_update`)
-
-
-### selfAddr
-
-```js
-window.webxdc.selfAddr
-```
-
-Property with the peer's own address.
-This is esp. useful if you want to differ between different peers -
-just send the address along with the payload,
-and, if needed, compare the payload addresses against selfAddr() later on.
-
-
-### selfName
-
-```js
-window.webxdc.selfName
-```
-
-Property with the peer's own name.
-This is name chosen by the user in their settings,
-if there is nothing set, that defaults to the peer's address.
-
-
-## manifest.toml
-
-If the ZIP-file contains a `manifest.toml` in its root directory,
-some basic information are read and used from there.
-
-the `manifest.toml` has the following format
-
-```toml
-name = "My App Name"
-```
-
-- **name** - The name of the app.
-  If no name is set or if there is no manifest, the filename is used as the app name.
-
-
-## App Icon
-
-If the ZIP-root contains an `icon.png` or `icon.jpg`,
-these files are used as the icon for the app.
-The icon should be a square at reasonable width/height;
-round corners etc. will be added by the implementations as needed.
-If no icon is set, a default icon will be used.
-
-
-## Webxdc Examples
-
-The following example shows an input field and  every input is show on all peers.
-
-```html
-<!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="utf-8"/>
-    <script src="webxdc.js"></script>
-  </head>
-  <body>
-    <input id="input" type="text"/>
-    <a href="" onclick="sendMsg(); return false;">Send</a>
-    <p id="output"></p>
-    <script>
-    
-      function sendMsg() {
-        msg = document.getElementById("input").value;
-        window.webxdc.sendUpdate({payload: msg}, 'Someone typed "'+msg+'".');
-      }
-    
-      function receiveUpdate(update) {
-        document.getElementById('output').innerHTML += update.payload + "<br>";
-      }
-    
-      window.webxdc.setUpdateListener(receiveUpdate, 0);
-    </script>
-  </body>
-</html>
-```
-
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
-You can also use that repository as a template for your own app -
-just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/cmdline.rs

@@ -2,7 +2,7 @@ extern crate dirs;
 
 use std::str::FromStr;
 
-use anyhow::{bail, ensure, Result};
+use anyhow::{bail, ensure, Error};
 use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
@@ -13,14 +13,15 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
+use deltachat::lot::LotState;
+use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
+use deltachat::EventType;
 use deltachat::{config, provider};
 use std::fs;
 use std::time::{Duration, SystemTime};
@@ -33,7 +34,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 1 {
         context
             .sql()
-            .execute("DELETE FROM jobs;", paramsv![])
+            .execute(sqlx::query("DELETE FROM jobs;"))
             .await
             .unwrap();
         println!("(1) Jobs reset.");
@@ -41,7 +42,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 2 {
         context
             .sql()
-            .execute("DELETE FROM acpeerstates;", paramsv![])
+            .execute(sqlx::query("DELETE FROM acpeerstates;"))
             .await
             .unwrap();
         println!("(2) Peerstates reset.");
@@ -49,7 +50,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 4 {
         context
             .sql()
-            .execute("DELETE FROM keypairs;", paramsv![])
+            .execute(sqlx::query("DELETE FROM keypairs;"))
             .await
             .unwrap();
         println!("(4) Private keypairs reset.");
@@ -57,48 +58,49 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 8 {
         context
             .sql()
-            .execute("DELETE FROM contacts WHERE id>9;", paramsv![])
+            .execute(sqlx::query("DELETE FROM contacts WHERE id>9;"))
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats WHERE id>9;", paramsv![])
+            .execute(sqlx::query("DELETE FROM chats WHERE id>9;"))
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats_contacts;", paramsv![])
+            .execute(sqlx::query("DELETE FROM chats_contacts;"))
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM msgs WHERE id>9;", paramsv![])
+            .execute(sqlx::query("DELETE FROM msgs WHERE id>9;"))
             .await
             .unwrap();
         context
             .sql()
-            .execute(
+            .execute(sqlx::query(
                 "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-                paramsv![],
-            )
+            ))
             .await
             .unwrap();
-        context.sql().config_cache().write().await.clear();
         context
             .sql()
-            .execute("DELETE FROM leftgrps;", paramsv![])
+            .execute(sqlx::query("DELETE FROM leftgrps;"))
             .await
             .unwrap();
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_msgs_changed_without_ids();
+    context.emit_event(EventType::MsgsChanged {
+        chat_id: ChatId::new(0),
+        msg_id: MsgId::new(0),
+    });
 }
 
-async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
+async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
     let data = dc_read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, false).await {
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false).await {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -160,7 +162,10 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_msgs_changed_without_ids();
+        context.emit_event(EventType::MsgsChanged {
+            chat_id: ChatId::new(0),
+            msg_id: MsgId::new(0),
+        });
     }
     true
 }
@@ -183,18 +188,10 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         MessageState::OutFailed => " !!",
         _ => "",
     };
-
-    let downloadstate = match msg.download_state() {
-        DownloadState::Done => "",
-        DownloadState::Available => " [⬇ Download available]",
-        DownloadState::InProgress => " [⬇ Download in progress...]️",
-        DownloadState::Failure => " [⬇ Download failed]",
-    };
-
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -203,7 +200,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == ContactId::SELF {
+        if msg.get_from_id() == 1 {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -228,12 +225,11 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             ""
         },
         statestr,
-        downloadstate,
         &temp2,
     );
 }
 
-async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
+async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
@@ -261,54 +257,59 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[u32]) {
     for contact_id in contacts {
+        let line;
         let mut line2 = "".to_string();
-        let contact = Contact::get_by_id(context, *contact_id).await?;
-        let name = contact.get_display_name();
-        let addr = contact.get_addr();
-        let verified_state = contact.is_verified(context).await?;
-        let verified_str = if VerifiedStatus::Unverified != verified_state {
-            if verified_state == VerifiedStatus::BidirectVerified {
-                " √√"
+        if let Ok(contact) = Contact::get_by_id(context, *contact_id).await {
+            let name = contact.get_display_name();
+            let addr = contact.get_addr();
+            let verified_state = contact.is_verified(context).await;
+            let verified_str = if VerifiedStatus::Unverified != verified_state {
+                if verified_state == VerifiedStatus::BidirectVerified {
+                    " √√"
+                } else {
+                    " √"
+                }
             } else {
-                " √"
-            }
-        } else {
-            ""
-        };
-        let line = format!(
-            "{}{} <{}>",
-            if !name.is_empty() {
-                name
-            } else {
-                "<name unset>"
-            },
-            verified_str,
-            if !addr.is_empty() { addr } else { "addr unset" }
-        );
-        let peerstate = Peerstate::from_addr(context, addr)
-            .await
-            .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != ContactId::SELF {
-            line2 = format!(
-                ", prefer-encrypt={}",
-                peerstate.as_ref().unwrap().prefer_encrypt
+                ""
+            };
+            line = format!(
+                "{}{} <{}>",
+                if !name.is_empty() {
+                    &name
+                } else {
+                    "<name unset>"
+                },
+                verified_str,
+                if !addr.is_empty() {
+                    &addr
+                } else {
+                    "addr unset"
+                }
             );
-        }
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
+            if peerstate.is_some() && *contact_id != 1 {
+                line2 = format!(
+                    ", prefer-encrypt={}",
+                    peerstate.as_ref().unwrap().prefer_encrypt
+                );
+            }
 
-        println!("Contact#{}: {}{}", *contact_id, line, line2);
+            println!("Contact#{}: {}{}", *contact_id, line, line2);
+        }
     }
-    Ok(())
 }
 
 fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<()> {
+pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Result<(), Error> {
     let mut sel_chat = if !chat_id.is_unset() {
-        Some(Chat::load_from_db(&context, *chat_id).await?)
+        Chat::load_from_db(&context, *chat_id).await.ok()
     } else {
         None
     };
@@ -349,7 +350,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -359,7 +359,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  chat [<chat-id>|0]\n\
                  createchat <contact-id>\n\
                  creategroup <name>\n\
-                 createbroadcast\n\
                  createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
@@ -372,11 +371,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  getlocations [<contact-id>]\n\
                  send <text>\n\
                  sendimage <file> [<text>]\n\
-                 sendsticker <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  sendhtml <file for html-part> [<text for plain-part>]\n\
-                 sendsyncmsg\n\
-                 sendupdate <msg-id> <json status update>\n\
                  videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
@@ -390,16 +386,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
-                 accept <chat-id>\n\
-                 decline <chat-id>\n\
+                 ===========================Contact requests==\n\
+                 decidestartchat <msg-id>\n\
+                 decideblock <msg-id>\n\
+                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 download <msg-id>\n\
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
-                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
@@ -414,12 +410,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listblocked\n\
                  ======================================Misc.==\n\
                  getqr [<chat-id>]\n\
-                 getqrsvg [<chat-id>]\n\
                  getbadqr\n\
                  checkqr <qr-content>\n\
-                 joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
+                 event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
                  clear -- clear screen\n\
@@ -454,39 +449,27 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(&context, MsgId::new(arg1.parse()?), arg2).await?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
         }
         "has-backup" => {
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(
-                &context,
-                ImexMode::ExportBackup,
-                dir.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ExportBackup, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            imex(
-                &context,
-                ImexMode::ImportBackup,
-                arg1.as_ref(),
-                Some(arg2.to_string()),
-            )
-            .await?;
+            imex(&context, ImexMode::ImportBackup, arg1).await?;
         }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
-            imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?;
+            imex(&context, ImexMode::ExportSelfKeys, &dir).await?;
             println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
+            imex(&context, ImexMode::ImportSelfKeys, arg1).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -513,33 +496,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
             context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
         "info" => {
             println!("{:#?}", context.get_info().await);
         }
-        "connectivity" => {
-            let file = dirs::home_dir()
-                .unwrap_or_default()
-                .join("connectivity.html");
-            match context.get_connectivity_html().await {
-                Ok(html) => {
-                    fs::write(&file, html)?;
-                    println!("Report written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get connectivity html: {}", err);
-                }
-            }
-        }
         "maybenetwork" => {
             context.maybe_network().await;
         }
@@ -565,9 +534,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 );
 
                 for i in (0..cnt).rev() {
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}{}{}",
+                        "{}#{}: {} [{} fresh] {}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -579,31 +548,27 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Pinned => "📌",
                         },
                         if chat.is_protected() { "🛡️" } else { "" },
-                        if chat.is_contact_request() {
-                            "🆕"
-                        } else {
-                            ""
-                        },
                     );
-                    let summary = chatlist.get_summary(&context, i, Some(&chat)).await?;
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
-                        match summary.state {
-                            MessageState::OutPending => " o",
-                            MessageState::OutDelivered => " √",
-                            MessageState::OutMdnRcvd => " √√",
-                            MessageState::OutFailed => " !!",
+                        match lot.get_state() {
+                            LotState::MsgOutPending => " o",
+                            LotState::MsgOutDelivered => " √",
+                            LotState::MsgOutMdnRcvd => " √√",
+                            LotState::MsgOutFailed => " !!",
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(summary.timestamp);
+                    let timestr = dc_timestamp_to_str(lot.get_timestamp());
+                    let text1 = lot.get_text1();
+                    let text2 = lot.get_text2();
                     println!(
-                        "{}{}{} [{}]{}",
-                        summary
-                            .prefix
-                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
-                        summary.text,
+                        "{}{}{}{} [{}]{}",
+                        text1.unwrap_or(""),
+                        if text1.is_some() { ": " } else { "" },
+                        text2.unwrap_or(""),
                         statestr,
                         &timestr,
                         if chat.is_sending_locations() {
@@ -617,7 +582,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await? {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -638,8 +603,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let time_start = std::time::SystemTime::now();
-            let msglist =
-                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
+            let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             let msglist: Vec<MsgId> = msglist
@@ -708,11 +672,40 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
-            let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
+            let contact_id: u32 = arg1.parse()?;
+            let chat_id = chat::create_by_contact_id(&context, contact_id).await?;
 
             println!("Single#{} created successfully.", chat_id,);
         }
+        "decidestartchat" | "createchatbymsg" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            match message::decide_on_contact_request(
+                &context,
+                msg_id,
+                ContactRequestDecision::StartChat,
+            )
+            .await
+            {
+                Some(chat_id) => {
+                    let chat = Chat::load_from_db(&context, chat_id).await?;
+                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
+                }
+                None => println!("Cannot crate chat."),
+            }
+        }
+        "decidenotnow" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
+                .await;
+        }
+        "decideblock" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
+                .await;
+        }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -720,11 +713,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createbroadcast" => {
-            let chat_id = chat::create_broadcast_list(&context).await?;
-
-            println!("Broadcast#{} created successfully.", chat_id);
-        }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -736,15 +724,23 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0 = ContactId::new(arg1.parse()?);
-            chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
-                .await?;
-            println!("Contact added to chat.");
+            let contact_id_0: u32 = arg1.parse()?;
+            if chat::add_contact_to_chat(
+                &context,
+                sel_chat.as_ref().unwrap().get_id(),
+                contact_id_0,
+            )
+            .await
+            {
+                println!("Contact added to chat.");
+            } else {
+                bail!("Cannot add contact to chat.");
+            }
         }
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1 = ContactId::new(arg1.parse()?);
+            let contact_id_1: u32 = arg1.parse()?;
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
@@ -757,12 +753,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "groupname" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            chat::set_chat_name(
-                &context,
-                sel_chat.as_ref().unwrap().get_id(),
-                format!("{} {}", arg1, arg2).trim(),
-            )
-            .await?;
+            chat::set_chat_name(&context, sel_chat.as_ref().unwrap().get_id(), arg1).await?;
 
             println!("Chat name set");
         }
@@ -782,7 +773,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 chat::get_chat_contacts(&context, sel_chat.as_ref().unwrap().get_id()).await?;
             println!("Memberlist:");
 
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!(
                 "{} contacts\nLocation streaming: {}",
                 contacts.len(),
@@ -790,7 +781,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     &context,
                     Some(sel_chat.as_ref().unwrap().get_id())
                 )
-                .await?,
+                .await,
             );
         }
         "getlocations" => {
@@ -835,7 +826,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 sel_chat.as_ref().unwrap().get_id(),
                 seconds,
             )
-            .await?;
+            .await;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -872,14 +863,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), "".into()).await?;
         }
-        "sendimage" | "sendsticker" | "sendfile" => {
+        "sendimage" | "sendfile" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No file given.");
 
             let mut msg = Message::new(if arg0 == "sendimage" {
                 Viewtype::Image
-            } else if arg0 == "sendsticker" {
-                Viewtype::Sticker
             } else {
                 Viewtype::File
             });
@@ -905,20 +894,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }));
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
-        "sendsyncmsg" => match context.send_sync_msg().await? {
-            Some(msg_id) => println!("sync message sent as {}.", msg_id),
-            None => println!("sync message not needed."),
-        },
-        "sendupdate" => {
-            ensure!(
-                !arg1.is_empty() && !arg2.is_empty(),
-                "Arguments <msg-id> <json status update> expected"
-            );
-            let msg_id = MsgId::new(arg1.parse()?);
-            context
-                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
-                .await?;
-        }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
@@ -926,24 +901,18 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let query = format!("{} {}", arg1, arg2).trim().to_string();
-            let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
+            let chat = if let Some(ref sel_chat) = sel_chat {
+                Some(sel_chat.get_id())
+            } else {
+                None
+            };
+
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, &query).await?;
+            let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!(
-                "{}{} messages for {}search of \"{}\"",
-                msglist.len(),
-                if msglist.len() == 1000 { "+" } else { "" },
-                if chat.is_none() {
-                    "global "
-                } else {
-                    "in-chat-"
-                },
-                query,
-            );
+            println!("{} messages.", msglist.len());
             println!("{:?} to create this list", time_needed);
         }
         "draft" => {
@@ -1051,28 +1020,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
-        "accept" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.accept(&context).await?;
-        }
-        "blockchat" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.block(&context).await?;
-        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
-        "download" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            println!("Scheduling download for {:?}", id);
-            id.download_full(&context).await?;
-        }
         "html" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1100,24 +1053,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
-        "resend" => {
-            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
-
-            let mut msg_ids = [MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            chat::resend_msgs(&context, &msg_ids).await?;
-        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
             msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::markseen_msgs(&context, msg_ids).await?;
+            message::markseen_msgs(&context, msg_ids).await;
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(&context, &ids).await?;
+            message::delete_msgs(&context, &ids).await;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
@@ -1130,7 +1076,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 Some(arg1),
             )
             .await?;
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} contacts.", contacts.len());
         }
         "addcontact" => {
@@ -1138,7 +1084,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if !arg2.is_empty() {
                 let book = format!("{}\n{}", arg1, arg2);
-                Contact::add_address_book(&context, &book).await?;
+                Contact::add_address_book(&context, book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
             }
@@ -1146,7 +1092,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id = ContactId::new(arg1.parse()?);
+            let contact_id: u32 = arg1.parse()?;
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
@@ -1172,7 +1118,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     if 0 != i {
                         res += ", ";
                     }
-                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)?).await?;
+                    let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     res += &format!("{}#{}", chat_prefix(&chat), chat.get_id());
                 }
             }
@@ -1181,27 +1127,33 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(&context, ContactId::new(arg1.parse()?)).await?;
+            Contact::delete(&context, arg1.parse()?).await?;
         }
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
-            Contact::block(&context, contact_id).await?;
+            let contact_id = arg1.parse()?;
+            Contact::block(&context, contact_id).await;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = ContactId::new(arg1.parse()?);
-            Contact::unblock(&context, contact_id).await?;
+            let contact_id = arg1.parse()?;
+            Contact::unblock(&context, contact_id).await;
         }
         "listblocked" => {
             let contacts = Contact::get_all_blocked(&context).await?;
-            log_contactlist(&context, &contacts).await?;
+            log_contactlist(&context, &contacts).await;
             println!("{} blocked contacts.", contacts.len());
         }
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            let qr = check_qr(&context, arg1).await?;
-            println!("qr={:?}", qr);
+            let res = check_qr(&context, arg1).await;
+            println!(
+                "state={}, id={}, text1={:?}, text2={:?}",
+                res.get_state(),
+                res.get_id(),
+                res.get_text1(),
+                res.get_text2()
+            );
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
@@ -1212,10 +1164,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            let socks5_enabled = context
-                .get_config_bool(config::Config::Socks5Enabled)
-                .await?;
-            match provider::get_provider_info(&context, arg1, socks5_enabled).await {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);
@@ -1231,6 +1180,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
         }
+        // TODO: implement this again, unclear how to match this through though, without writing a parser.
+        // "event" => {
+        //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
+        //     let event = arg1.parse()?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
+        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
+        //     println!(
+        //         "Sending event {:?}({}), received value {}.",
+        //         event, event as usize, r,
+        //     );
+        // }
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 

examples/repl/main.rs

@@ -26,15 +26,12 @@ use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
-use rustyline::validate::Validator;
 use rustyline::{
-    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
+    Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyPress,
 };
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -59,6 +56,9 @@ fn receive_event(event: EventType) {
         EventType::Error(msg) => {
             error!("{}", msg);
         }
+        EventType::ErrorNetwork(msg) => {
+            error!("[NETWORK] msg={}", msg);
+        }
         EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
@@ -156,7 +156,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
     "set",
     "get",
@@ -164,19 +164,20 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 36] = [
+const CHAT_COMMANDS: [&str; 34] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "decidestartchat",
+    "decideblock",
+    "decidenotnow",
     "creategroup",
-    "createbroadcast",
-    "createprotected",
+    "createverified",
     "addmember",
     "removemember",
     "groupname",
@@ -190,8 +191,6 @@ const CHAT_COMMANDS: [&str; 36] = [
     "sendimage",
     "sendfile",
     "sendhtml",
-    "sendsyncmsg",
-    "sendupdate",
     "videochat",
     "draft",
     "listmedia",
@@ -204,18 +203,14 @@ const CHAT_COMMANDS: [&str; 36] = [
     "protect",
     "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
-    "resend",
     "markseen",
     "delmsg",
-    "download",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -228,12 +223,11 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 11] = [
+const MISC_COMMANDS: [&str; 10] = [
     "getqr",
-    "getqrsvg",
     "getbadqr",
     "checkqr",
-    "joinqr",
+    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -243,9 +237,7 @@ const MISC_COMMANDS: [&str; 11] = [
 ];
 
 impl Hinter for DcHelper {
-    type Hint = String;
-
-    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<Self::Hint> {
+    fn hint(&self, line: &str, pos: usize, ctx: &RustyContext<'_>) -> Option<String> {
         if !line.is_empty() {
             for &cmds in &[
                 &IMEX_COMMANDS[..],
@@ -267,10 +259,11 @@ impl Hinter for DcHelper {
 }
 
 static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
-    fn highlight_prompt<'b, 's: 'b, 'p: 'b>(&self, prompt: &'p str, default: bool) -> Cow<'b, str> {
-        if default {
+    fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
+        if prompt == PROMPT {
             Borrowed(COLORED_PROMPT)
         } else {
             Borrowed(prompt)
@@ -291,14 +284,13 @@ impl Highlighter for DcHelper {
 }
 
 impl Helper for DcHelper {}
-impl Validator for DcHelper {}
 
 async fn start(args: Vec<String>) -> Result<(), Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
@@ -325,15 +317,15 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         };
         let mut rl = Editor::with_config(config);
         rl.set_helper(Some(h));
-        rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
         if rl.load_history(".dc-history.txt").is_err() {
             println!("No previous history.");
         }
 
         loop {
             let p = "> ";
-            let readline = rl.readline(p);
+            let readline = rl.readline(&p);
 
             match readline {
                 Ok(line) => {
@@ -400,7 +392,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -416,32 +408,19 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
-            if !qr.is_empty() {
-                if arg0 == "getbadqr" && qr.len() > 40 {
-                    qr.replace_range(12..22, "0000000000")
-                }
-                println!("{}", qr);
-                let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
-                    .output()
-                    .expect("failed to execute process");
-                io::stdout().write_all(&output.stdout).unwrap();
-                io::stderr().write_all(&output.stderr).unwrap();
-            }
-        }
-        "getqrsvg" => {
-            ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
-            let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
-            match get_securejoin_qr_svg(&ctx, group).await {
-                Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get QR code svg: {}", err);
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
+                if !qr.is_empty() {
+                    if arg0 == "getbadqr" && qr.len() > 40 {
+                        qr.replace_range(12..22, "0000000000")
+                    }
+                    println!("{}", qr);
+                    let output = Command::new("qrencode")
+                        .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                        .output()
+                        .expect("failed to execute process");
+                    io::stdout().write_all(&output.stdout).unwrap();
+                    io::stderr().write_all(&output.stderr).unwrap();
                 }
             }
         }

examples/simple.rs

@@ -1,6 +1,6 @@
 use tempfile::tempdir;
 
-use deltachat::chat::{self, ChatId};
+use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
@@ -19,7 +19,7 @@ fn cb(event: EventType) {
         EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
@@ -70,7 +70,7 @@ async fn main() {
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
     for i in 0..1 {
         log::info!("sending message {}", i);
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

proptest-regressions/dc_tools.txt

@@ -7,5 +7,3 @@
 cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
 cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
 cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ  ", approx_chars = 0, do_unwrap = false
-cc 9796807baeda701227dcdcfc9fdaa93ddd556da2bb1630381bfe2e037bee73f6 # shrinks to buf = " ꫛ®a\u{11300}a", approx_chars = 0
-cc 063a4c42ac1ec9aa37af54521b210ba9cd82dcc9cc3be296ca2fedf8240072d4 # shrinks to buf = "a᪠  0A", approx_chars = 0

python/README.rst

@@ -58,13 +58,12 @@ end-to-end tests that require accounts on real e-mail servers.
 running "live" tests with temporary accounts
 ---------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
 
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
 One hour is enough to invoke pytest and run all offline and online tests:
 
     pytest

python/examples/test_examples.py

@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1, = acfactory.get_online_accounts(1)
+    ac1 = acfactory.get_one_online_account()
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -42,7 +42,7 @@ def test_group_tracking_plugin(acfactory, lp):
     lp.sec("creating one group-tracking bot and two temp accounts")
     botproc = acfactory.run_bot_process(group_tracking, ffi=False)
 
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
 
     botproc.fnmatch_lines("""
         *ac_configure_completed*

python/install_python_bindings.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
         subprocess.check_call([

python/mypy.ini

@@ -1,23 +0,0 @@
-[mypy]
-
-[mypy-deltachat.capi.*]
-ignore_missing_imports = True
-
-[mypy-pluggy.*]
-ignore_missing_imports = True
-
-[mypy-cffi.*]
-ignore_missing_imports = True
-
-[mypy-imapclient.*]
-ignore_missing_imports = True
-
-[mypy-pytest.*]
-ignore_missing_imports = True
-
-[mypy-_pytest.*]
-ignore_missing_imports = True
-
-[mypy-imap_tools.*]
-ignore_missing_imports = True
-

python/pyproject.toml

@@ -1,8 +0,0 @@
-[build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"

python/setup.py

@@ -8,14 +8,17 @@ def main():
         long_description = f.read()
     setuptools.setup(
         name='deltachat',
+        setup_requires=['setuptools_scm', 'cffi>=1.0.0'],
+        use_scm_version = {
+            "root": "..",
+            "relative_to": __file__,
+            'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
+        },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
-        setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
-        ],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -2,7 +2,7 @@ import sys
 
 from . import capi, const, hookspec # noqa
 from .capi import ffi  # noqa
-from .account import Account, get_core_info  # noqa
+from .account import Account  # noqa
 from .message import Message  # noqa
 from .contact import Contact  # noqa
 from .chat import Chat        # noqa
@@ -19,9 +19,9 @@ except DistributionNotFound:
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
-        for name in dir(const):
+        for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
-                _DC_EVENTNAME_MAP[getattr(const, name)] = name
+                _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
 
 
@@ -75,6 +75,7 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
         ac.set_config("bot", "1")
         configtracker = ac.configure()

python/src/deltachat/_build.py

@@ -8,9 +8,11 @@ import shutil
 import subprocess
 import tempfile
 import textwrap
+import types
+from os.path import abspath
+from os.path import dirname as dn
 
 import cffi
-import pkgconfig  # type: ignore
 
 
 def local_build_flags(projdir, target):
@@ -19,31 +21,35 @@ def local_build_flags(projdir, target):
     :param projdir: The root directory of the deltachat-core-rust project.
     :param target: The rust build target, `debug` or `release`.
     """
-    flags = {}
+    flags = types.SimpleNamespace()
     if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
                 '-framework', 'CoreFoundation',
                 '-framework', 'CoreServices',
                 '-framework', 'Security',
         ]
     elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
         target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
 
 
 def extract_functions(flags):
@@ -64,7 +70,7 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
@@ -100,7 +106,7 @@ def find_header(flags):
         try:
             os.chdir(tmpdir)
             cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
+                       include_dirs=flags.incs,
                        macros=[("PY_CFFI_INC", "1")])
         finally:
             os.chdir(cwd)
@@ -145,7 +151,6 @@ def extract_defines(flags):
                 | DC_PROVIDER
                 | DC_KEY_GEN
                 | DC_IMEX
-                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains
@@ -163,8 +168,11 @@ def extract_defines(flags):
 
 def ffibuilder():
     projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()
@@ -178,7 +186,10 @@ def ffibuilder():
                 return DC_EVENT_DATA2_IS_STRING(e);
             }
         """,
-        **flags,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
     )
     builder.cdef("""
         typedef int... time_t;

python/src/deltachat/account.py

@@ -8,43 +8,19 @@ import os
 from array import array
 from . import const
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
 from .tracker import ImexTracker, ConfigureTracker
 from . import hookspec
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
 
 
 class MissingCredentials(ValueError):
     """ Account is missing `addr` and `mail_pw` config values. """
 
 
-def get_core_info():
-    """ get some system info. """
-    from tempfile import NamedTemporaryFile
-
-    with NamedTemporaryFile() as path:
-        path.close()
-        return get_dc_info_as_dict(ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
-            lib.dc_context_unref,
-        ))
-
-
-def get_dc_info_as_dict(dc_context):
-    lines = from_dc_charpointer(lib.dc_get_info(dc_context))
-    info_dict = {}
-    for line in lines.split("\n"):
-        if not line.strip():
-            continue
-        key, value = line.split("=", 1)
-        info_dict[key.lower()] = value
-    return info_dict
-
-
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
@@ -52,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
+    def __init__(self, db_path, os_name=None, logging=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -82,17 +58,14 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
-    def disable_logging(self) -> None:
+    def disable_logging(self):
         """ disable logging. """
         self._logging = False
 
-    def enable_logging(self) -> None:
+    def enable_logging(self):
         """ re-enable logging. """
         self._logging = True
 
-    def __repr__(self):
-        return "<Account path={}>".format(self.db_path)
-
     # def __del__(self):
     #    self.shutdown()
 
@@ -100,14 +73,21 @@ class Account(object):
         if self._logging:
             self._pm.hook.ac_log_line(message=msg)
 
-    def _check_config_key(self, name: str) -> None:
+    def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
                            name, self._configkeys))
 
-    def get_info(self) -> Dict[str, str]:
+    def get_info(self):
         """ return dictionary of built config parameters. """
-        return get_dc_info_as_dict(self._dc_context)
+        lines = from_dc_charpointer(lib.dc_get_info(self._dc_context))
+        d = {}
+        for line in lines.split("\n"):
+            if not line.strip():
+                continue
+            key, value = line.split("=", 1)
+            d[key.lower()] = value
+        return d
 
     def dump_account_info(self, logfile):
         def log(*args, **kwargs):
@@ -125,19 +105,19 @@ class Account(object):
             cursor += len(entry) + 1
         log("")
 
-    def set_stock_translation(self, id: int, string: str) -> None:
+    def set_stock_translation(self, id, string):
         """ set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
         :returns: None
         """
-        bytestring = string.encode("utf8")
-        res = lib.dc_set_stock_translation(self._dc_context, id, bytestring)
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
         if res == 0:
             raise ValueError("could not set translation string")
 
-    def set_config(self, name: str, value: Optional[str]) -> None:
+    def set_config(self, name, value):
         """ set configuration values.
 
         :param name: config key name (unicode)
@@ -145,18 +125,16 @@ class Account(object):
         :returns: None
         """
         self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
+        name = name.encode("utf8")
+        if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
-        if isinstance(value, (int, bool)):
-            value = str(int(value))
         if value is not None:
-            valuebytes = value.encode("utf8")
+            value = value.encode("utf8")
         else:
-            valuebytes = ffi.NULL
-        lib.dc_set_config(self._dc_context, namebytes, valuebytes)
+            value = ffi.NULL
+        lib.dc_set_config(self._dc_context, name, value)
 
-    def get_config(self, name: str) -> str:
+    def get_config(self, name):
         """ return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
@@ -165,12 +143,12 @@ class Account(object):
         """
         if name != "sys.config_keys":
             self._check_config_key(name)
-        namebytes = name.encode("utf8")
-        res = lib.dc_get_config(self._dc_context, namebytes)
+        name = name.encode("utf8")
+        res = lib.dc_get_config(self._dc_context, name)
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
+    def _preconfigure_keypair(self, addr, public, secret):
         """See dc_preconfigure_keypair() in deltachat.h.
 
         In other words, you don't need this.
@@ -182,7 +160,7 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs: Dict[str, Any]) -> None:
+    def update_config(self, kwargs):
         """ update config values.
 
         :param kwargs: name=value config settings for this account.
@@ -190,9 +168,9 @@ class Account(object):
         :returns: None
         """
         for key, value in kwargs.items():
-            self.set_config(key, value)
+            self.set_config(key, str(value))
 
-    def is_configured(self) -> bool:
+    def is_configured(self):
         """ determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
@@ -200,13 +178,7 @@ class Account(object):
         """
         return True if lib.dc_is_configured(self._dc_context) else False
 
-    def is_open(self) -> bool:
-        """Determine if account is open
-
-        :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
-
-    def set_avatar(self, img_path: Optional[str]) -> None:
+    def set_avatar(self, img_path):
         """Set self avatar.
 
         :raises ValueError: if profile image could not be set
@@ -218,18 +190,20 @@ class Account(object):
             assert os.path.exists(img_path), img_path
             self.set_config("selfavatar", img_path)
 
-    def check_is_configured(self) -> None:
+    def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_latest_backupfile(self, backupdir) -> Optional[str]:
+    def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
-        return from_optional_dc_charpointer(res)
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
-    def get_blobdir(self) -> str:
+    def get_blobdir(self):
         """ return the directory for files.
 
         All sent files are copied to this directory if necessary.
@@ -237,15 +211,15 @@ class Account(object):
         """
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
-    def get_self_contact(self) -> Contact:
+    def get_self_contact(self):
         """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name: Optional[str] = None) -> Contact:
-        """create a (new) Contact or return an existing one.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
         Calling this method will always result in the same
         underlying contact id.  If there already is a Contact
@@ -262,13 +236,13 @@ class Account(object):
         contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
-    def get_contact(self, obj) -> Optional[Contact]:
+    def get_contact(self, obj):
         if isinstance(obj, Contact):
             return obj
         (_, addr) = self.get_contact_addr_and_name(obj)
         return self.get_contact_by_addr(addr)
 
-    def get_contact_addr_and_name(self, obj, name: Optional[str] = None):
+    def get_contact_addr_and_name(self, obj, name=None):
         if isinstance(obj, Account):
             if not obj.is_configured():
                 raise ValueError("can only add addresses from configured accounts")
@@ -286,7 +260,7 @@ class Account(object):
             name = displayname
         return (name, addr)
 
-    def delete_contact(self, contact: Contact) -> bool:
+    def delete_contact(self, contact):
         """ delete a Contact.
 
         :param contact: contact object obtained
@@ -297,23 +271,22 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email: str) -> Optional[Contact]:
+    def get_contact_by_addr(self, email):
         """ get a contact for the email address or None if it's blocked or doesn't exist. """
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
         if contact_id:
             return self.get_contact_by_id(contact_id)
-        return None
 
-    def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
         :param contact_id: integer id of this contact.
-        :returns: :class:`deltachat.contact.Contact` instance.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
-    def get_blocked_contacts(self) -> List[Contact]:
+    def get_blocked_contacts(self):
         """ return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
@@ -324,13 +297,8 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_contacts(
-        self,
-        query: Optional[str] = None,
-        with_self: bool = False,
-        only_verified: bool = False,
-    ) -> List[Contact]:
-        """get a (filtered) list of contacts.
+    def get_contacts(self, query=None, with_self=False, only_verified=False):
+        """ get a (filtered) list of contacts.
 
         :param query: if a string is specified, only return contacts
                       whose name or e-mail matches query.
@@ -350,7 +318,7 @@ class Account(object):
         )
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
-    def get_fresh_messages(self) -> Generator[Message, None, None]:
+    def get_fresh_messages(self):
         """ yield all fresh messages from all chats. """
         dc_array = ffi.gc(
             lib.dc_get_fresh_msgs(self._dc_context),
@@ -358,17 +326,15 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj) -> Chat:
+    def create_chat(self, obj):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
-    def create_group_chat(
-        self,
-        name: str,
-        contacts: Optional[List[Contact]] = None,
-        verified: bool = False,
-    ) -> Chat:
-        """create a new group chat object.
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+
+    def create_group_chat(self, name, contacts=None, verified=False):
+        """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
@@ -384,7 +350,7 @@ class Account(object):
                 chat.add_contact(contact)
         return chat
 
-    def get_chats(self) -> List[Chat]:
+    def get_chats(self):
         """ return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
@@ -401,17 +367,20 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
-    def get_device_chat(self) -> Chat:
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
+
+    def get_device_chat(self):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
-    def get_chat_by_id(self, chat_id: int) -> Chat:
+    def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
@@ -423,21 +392,19 @@ class Account(object):
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
-    def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
+    def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
         arr = array("i")
         for msg in messages:
-            if isinstance(msg, Message):
-                arr.append(msg.id)
-            else:
-                arr.append(msg)
+            msg = getattr(msg, "id", msg)
+            arr.append(msg)
         msg_ids = ffi.cast("uint32_t*", ffi.from_buffer(arr))
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
-    def forward_messages(self, messages: List[Message], chat: Chat) -> None:
+    def forward_messages(self, messages, chat):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -447,7 +414,7 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages):
         """ delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
@@ -505,7 +472,7 @@ class Account(object):
     def imex(self, path, imex_cmd):
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
 
-    def initiate_key_transfer(self) -> str:
+    def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
         has been successfully sent to our own e-mail address ("self-sent message").
         If sending out was unsuccessful, a RuntimeError is raised.
@@ -516,7 +483,7 @@ class Account(object):
             raise RuntimeError("could not send out autocrypt setup message")
         return from_dc_charpointer(res)
 
-    def get_setup_contact_qr(self) -> str:
+    def get_setup_contact_qr(self):
         """ get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -566,9 +533,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -590,8 +555,6 @@ class Account(object):
         """ add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
-        if name and self._pm.has_plugin(name=name):
-            self._pm.unregister(name=name)
         self._pm.register(plugin, name=name)
         self._pm.check_pending()
         return plugin
@@ -611,15 +574,6 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self) -> str:
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 
@@ -664,7 +618,7 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
+    def configure(self, reconfigure=False):
         """ Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
@@ -677,11 +631,11 @@ class Account(object):
         lib.dc_configure(self._dc_context)
         return configtracker
 
-    def wait_shutdown(self) -> None:
+    def wait_shutdown(self):
         """ wait until shutdown of this account has completed. """
         self._shutdown_event.wait()
 
-    def stop_io(self) -> None:
+    def stop_io(self):
         """ stop core IO scheduler if it is running. """
         self.log("stop_ongoing")
         self.stop_ongoing()
@@ -689,7 +643,7 @@ class Account(object):
         self.log("dc_stop_io (stop core IO scheduler)")
         lib.dc_stop_io(self._dc_context)
 
-    def shutdown(self) -> None:
+    def shutdown(self):
         """ shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:
@@ -699,7 +653,14 @@ class Account(object):
 
         self.log("remove dc_context references")
 
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
         self._event_thread.mark_shutdown()
+        self._dc_context = None
 
         self.log("wait for event thread to finish")
         try:
@@ -707,8 +668,6 @@ class Account(object):
         except RuntimeError as e:
             self.log("Waiting for event thread failed: {}".format(e))
 
-        self._dc_context = None
-
         if self._event_thread.is_alive():
             self.log("WARN: event thread did not terminate yet, ignoring.")
 

python/src/deltachat/chat.py

@@ -3,13 +3,12 @@
 import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
+from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
-from typing import Optional
 
 
 class Chat(object):
@@ -18,20 +17,20 @@ class Chat(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
+    def __init__(self, account, id):
         from .account import Account
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
                self.account._dc_context == other.account._dc_context
 
-    def __ne__(self, other) -> bool:
+    def __ne__(self, other):
         return not (self == other)
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return "<Chat id={} name={}>".format(self.id, self.get_name())
 
     @property
@@ -41,7 +40,7 @@ class Chat(object):
             lib.dc_chat_unref
         )
 
-    def delete(self) -> None:
+    def delete(self):
         """Delete this chat and all its messages.
 
         Note:
@@ -51,37 +50,29 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self) -> None:
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self) -> None:
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self) -> bool:
+    def is_group(self):
         """ return true if this chat is a group chat.
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
-    def is_muted(self) -> bool:
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
+    def is_muted(self):
         """ return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -91,38 +82,38 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def can_send(self) -> bool:
+    def can_send(self):
         """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+        This is not true eg. for the deaddrop or for the device-talk
 
         :returns: True if the chat is writable, False otherwise
         """
         return lib.dc_chat_can_send(self._dc_chat)
 
-    def is_protected(self) -> bool:
+    def is_protected(self):
         """ return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
         return lib.dc_chat_is_protected(self._dc_chat)
 
-    def get_name(self) -> Optional[str]:
+    def get_name(self):
         """ return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
-    def set_name(self, name: str) -> bool:
+    def set_name(self, name):
         """ set name of this chat.
 
         :param name: as a unicode string.
-        :returns: True on success, False otherwise
+        :returns: None
         """
         name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
 
-    def mute(self, duration: Optional[int] = None) -> None:
+    def mute(self, duration=None):
         """ mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
@@ -136,7 +127,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
-    def unmute(self) -> None:
+    def unmute(self):
         """ unmutes the chat
 
         :returns: None
@@ -145,7 +136,7 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
-    def get_mute_duration(self) -> int:
+    def get_mute_duration(self):
         """ Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
@@ -153,37 +144,37 @@ class Chat(object):
         """
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
-    def get_ephemeral_timer(self) -> int:
+    def get_ephemeral_timer(self):
         """ get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
-    def set_ephemeral_timer(self, timer: int) -> bool:
+    def set_ephemeral_timer(self, timer):
         """ set ephemeral timer.
 
         :param: timer value in seconds
 
-        :returns: True on success, False otherwise
+        :returns: None
         """
-        return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
-    def get_type(self) -> int:
+    def get_type(self):
         """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
         return lib.dc_chat_get_type(self._dc_chat)
 
-    def get_encryption_info(self) -> Optional[str]:
+    def get_encryption_info(self):
         """Return encryption info for this chat.
 
         :returns: a string with encryption preferences of all chat members"""
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
-    def get_join_qr(self) -> Optional[str]:
+    def get_join_qr(self):
         """ get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
@@ -195,7 +186,7 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
-    def send_msg(self, msg: Message) -> Message:
+    def send_msg(self, msg):
         """send a message by using a ready Message object.
 
         :param msg: a :class:`deltachat.message.Message` instance
@@ -513,11 +504,10 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
+                timestamp=datetime.utcfromtimestamp(
+                    lib.dc_array_get_timestamp(dc_array, i)
                 ),
-                marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
+                marker=from_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))
         ]

python/src/deltachat/const.py

@@ -1,13 +1,7 @@
-from typing import Any, List
-
 from .capi import lib
 
 
-def __getattr__(name: str) -> Any:
+for name in dir(lib):
     if name.startswith("DC_"):
-        return getattr(lib, name)
-    return globals()[name]
-
-
-def __dir__() -> List[str]:
-    return sorted(name for name in dir(lib) if name.startswith("DC_"))
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -1,12 +1,10 @@
 """ Contact object. """
 
-from datetime import date, datetime, timezone
-from typing import Optional
-
-from . import const, props
-from .capi import ffi, lib
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
 from .chat import Chat
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer
+from . import const
 
 
 class Contact(object):
@@ -37,25 +35,18 @@ class Contact(object):
         )
 
     @props.with_doc
-    def addr(self) -> str:
+    def addr(self):
         """ normalized e-mail address for this account. """
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self) -> str:
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
     display_name = name
 
-    @props.with_doc
-    def last_seen(self) -> date:
-        """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -76,13 +67,15 @@ class Contact(object):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
 
-    def get_profile_image(self) -> Optional[str]:
+    def get_profile_image(self):
         """Get contact profile image.
 
         :returns: path to profile image, None if no profile image exists.
         """
         dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
-        return from_optional_dc_charpointer(dc_res)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
 
     @property
     def status(self):

python/src/deltachat/cutil.py

@@ -1,9 +1,6 @@
 from .capi import lib
 from .capi import ffi
-from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
-
-T = TypeVar('T')
+from datetime import datetime
 
 
 def as_dc_charpointer(obj):
@@ -14,28 +11,21 @@ def as_dc_charpointer(obj):
     return obj
 
 
-def iter_array(dc_array_t, constructor: Callable[[int], T]) -> Generator[T, None, None]:
+def iter_array(dc_array_t, constructor):
     for i in range(0, lib.dc_array_get_cnt(dc_array_t)):
         yield constructor(lib.dc_array_get_id(dc_array_t, i))
 
 
-def from_dc_charpointer(obj) -> str:
+def from_dc_charpointer(obj):
     if obj != ffi.NULL:
         return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    raise ValueError
-
-
-def from_optional_dc_charpointer(obj) -> Optional[str]:
-    if obj != ffi.NULL:
-        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
-    return None
 
 
 class DCLot:
-    def __init__(self, dc_lot) -> None:
+    def __init__(self, dc_lot):
         self._dc_lot = dc_lot
 
-    def id(self) -> int:
+    def id(self):
         return lib.dc_lot_get_id(self._dc_lot)
 
     def state(self):
@@ -54,4 +44,4 @@ class DCLot:
         ts = lib.dc_lot_get_timestamp(self._dc_lot)
         if ts == 0:
             return None
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)

python/src/deltachat/direct_imap.py

@@ -4,22 +4,65 @@ and for cleaning up inbox/mvbox for each test function run.
 """
 
 import io
+import email
 import ssl
 import pathlib
-from contextlib import contextmanager
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
 import imaplib
-from deltachat import const, Account
-from typing import List
+import deltachat
+from deltachat import const
 
 
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
 FLAGS = b'FLAGS'
 FETCH = b'FETCH'
 ALL = "1:*"
 
 
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    try:
+
+        if not hasattr(account, "direct_imap"):
+            imap = DirectImap(account)
+
+            for folder in imap.list_folders():
+                if folder.lower() == "inbox" or folder.lower() == "deltachat":
+                    assert imap.select_folder(folder)
+                    imap.delete(ALL, expunge=True)
+                else:
+                    imap.conn.delete_folder(folder)
+                    # We just deleted the folder, so we have to make DC forget about it, too
+                    if account.get_config("configured_sentbox_folder") == folder:
+                        account.set_config("configured_sentbox_folder", None)
+                    if account.get_config("configured_spam_folder") == folder:
+                        account.set_config("configured_spam_folder", None)
+
+            setattr(account, "direct_imap", imap)
+
+    except Exception as e:
+        # Uncaught exceptions here would lead to a timeout without any note written to the log
+        # start with DC_EVENT_WARNING so that the line is printed in yellow and won't be overlooked when reading
+        account.log("DC_EVENT_WARNING =================== DIRECT_IMAP CAN'T RESET ACCOUNT: ===================")
+        account.log("DC_EVENT_WARNING =================== " + str(e) + " ===================")
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
 class DirectImap:
-    def __init__(self, account: Account) -> None:
+    def __init__(self, account):
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
         self._idling = False
@@ -45,30 +88,37 @@ class DirectImap:
             ssl_context.verify_mode = ssl.CERT_NONE
 
         if security == const.DC_SOCKET_STARTTLS:
-            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
-        elif security == const.DC_SOCKET_PLAIN or security == const.DC_SOCKET_SSL:
-            self.conn = MailBox(host, port, ssl_context=ssl_context)
+            self.conn = IMAPClient(host, port, ssl=False)
+            self.conn.starttls(ssl_context)
+        elif security == const.DC_SOCKET_PLAIN:
+            self.conn = IMAPClient(host, port, ssl=False)
+        elif security == const.DC_SOCKET_SSL:
+            self.conn = IMAPClient(host, port, ssl_context=ssl_context)
         self.conn.login(user, pw)
 
         self.select_folder("INBOX")
 
     def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
         try:
             self.conn.logout()
-        except (OSError, imaplib.IMAP4.abort):
+        except (OSError, IMAPClientError):
             print("Could not logout direct_imap conn")
 
     def create_folder(self, foldername):
         try:
-            self.conn.folder.create(foldername)
-        except errors.MailboxFolderCreateError as e:
+            self.conn.create_folder(foldername)
+        except imaplib.IMAP4.error as e:
             print("Can't create", foldername, "probably it already exists:", str(e))
 
-    def select_folder(self, foldername: str) -> tuple:
+    def select_folder(self, foldername):
         assert not self._idling
-        return self.conn.folder.set(foldername)
+        return self.conn.select_folder(foldername)
 
-    def select_config_folder(self, config_name: str):
+    def select_config_folder(self, config_name):
         """ Return info about selected folder if it is
         configured, otherwise None. """
         if "_" not in config_name:
@@ -77,36 +127,50 @@ class DirectImap:
         if foldername:
             return self.select_folder(foldername)
 
-    def list_folders(self) -> List[str]:
+    def list_folders(self):
         """ return list of all existing folder names"""
         assert not self._idling
-        return [folder.name for folder in self.conn.folder.list()]
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
 
-    def delete(self, uid_list: str, expunge=True):
+    def delete(self, range, expunge=True):
         """ delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.set_flags(range, [DELETED])
         if expunge:
             self.conn.expunge()
 
-    def get_all_messages(self) -> List[MailMessage]:
+    def get_all_messages(self):
         assert not self._idling
-        return [mail for mail in self.conn.fetch()]
 
-    def get_unread_messages(self) -> List[str]:
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
         assert not self._idling
-        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
 
     def mark_all_read(self):
         messages = self.get_unread_messages()
         if messages:
-            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
+            res = self.conn.set_flags(messages, [SEEN])
             print("marked seen:", messages, res)
 
-    def get_unread_cnt(self) -> int:
+    def get_unread_cnt(self):
         return len(self.get_unread_messages())
 
     def dump_imap_structures(self, dir, logfile):
@@ -128,81 +192,66 @@ class DirectImap:
             log("---------", imapfolder, len(messages), "messages ---------")
             # get message content without auto-marking it as seen
             # fetching 'RFC822' would mark it as seen.
-            for msg in self.conn.fetch(mark_seen=False):
-                body = getattr(msg.obj, "text", None)
-                if not body:
-                    body = getattr(msg.obj, "html", None)
-                if not body:
-                    log("Message", msg.uid, "has empty body")
+            requested = [b'BODY.PEEK[]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
                     continue
 
+                flags = data[FLAGS]
                 path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(msg.uid))
-                fn.write_bytes(body)
-                log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
 
         print(stream.getvalue(), file=logfile)
 
-    @contextmanager
-    def idle(self):
-        """ return Idle ContextManager. """
-        idle_manager = IdleManager(self)
-        try:
-            yield idle_manager
-        finally:
-            idle_manager.done()
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
 
-    def append(self, folder: str, msg: str):
-        """Upload a message to *folder*.
-        Trailing whitespace or a linebreak at the beginning will be removed automatically.
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
         """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res
+
+    def append(self, folder, msg):
         if msg.startswith("\n"):
             msg = msg[1:]
         msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
-
-    def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
-        if len(msgs) == 0:
-            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
-        return msgs[0]
-
-
-class IdleManager:
-    def __init__(self, direct_imap):
-        self.direct_imap = direct_imap
-        self.log = direct_imap.account.log
-        self.direct_imap.conn.idle.start()
-
-    def check(self, timeout=None) -> List[bytes]:
-        """ (blocking) wait for next idle message from server. """
-        self.log("imap-direct: calling idle_check")
-        res = self.direct_imap.conn.idle.poll(timeout=timeout)
-        self.log("imap-direct: idle_check returned {!r}".format(res))
-        return res
-
-    def wait_for_new_message(self, timeout=None) -> bytes:
-        while 1:
-            for item in self.check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
-                    return item
-
-    def wait_for_seen(self, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
-        """
-        while 1:
-            for item in self.check(timeout=timeout):
-                if FETCH in item:
-                    self.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        return int(item.split(b' ')[1])
-
-    def done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        res = self.direct_imap.conn.idle.stop()
-        return res
+        self.conn.append(folder, msg)

python/src/deltachat/events.py

@@ -9,11 +9,11 @@ from .hookspec import account_hookimpl
 from contextlib import contextmanager
 from .capi import ffi, lib
 from .message import map_system_message
-from .cutil import from_optional_dc_charpointer
+from .cutil import from_dc_charpointer
 
 
 class FFIEvent:
-    def __init__(self, name: str, data1, data2):
+    def __init__(self, name, data1, data2):
         self.name = name
         self.data1 = data1
         self.data2 = data2
@@ -29,20 +29,18 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid=None, init_time=None) -> None:
+    def __init__(self, account):
         self.account = account
-        self.logid = logid or self.account.get_config("displayname")
-        if init_time is None:
-            init_time = time.time()
-        self.init_time = init_time
+        self.logid = self.account.get_config("displayname")
+        self.init_time = time.time()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent) -> None:
+    def ac_process_ffi_event(self, ffi_event):
         self.account.log(str(ffi_event))
 
     @account_hookimpl
     def ac_log_line(self, message):
-        t = threading.current_thread()
+        t = threading.currentThread()
         tname = getattr(t, "name", t)
         if tname == "MainThread":
             tname = "MAIN"
@@ -71,7 +69,7 @@ class FFIEventTracker:
         self._event_queue = Queue()
 
     @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event: FFIEvent):
+    def ac_process_ffi_event(self, ffi_event):
         self._event_queue.put(ffi_event)
 
     def set_timeout(self, timeout):
@@ -98,7 +96,7 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_contains(self, regex: str) -> FFIEvent:
+    def get_info_contains(self, regex):
         rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
@@ -113,33 +111,6 @@ class FFIEventTracker:
             if m is not None:
                 return m.groups()
 
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))
@@ -158,14 +129,14 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
-    def wait_idle_inbox_ready(self):
+    def wait_all_initial_fetches(self):
         """Has to be called after start_io() to wait for fetch_existing_msgs to run
         so that new messages are not mistaken for old ones:
         - ac1 and ac2 are created
         - ac1 sends a message to ac2
         - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
         - therefore no DC_EVENT_INCOMING_MSG is sent"""
-        self.get_info_contains("INBOX: Idle entering")
+        self.get_info_contains("Done fetching existing messages")
 
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
@@ -178,7 +149,6 @@ class FFIEventTracker:
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
-        return None
 
     def wait_msg_delivered(self, msg):
         ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -192,11 +162,11 @@ class EventThread(threading.Thread):
 
     With each Account init this callback thread is started.
     """
-    def __init__(self, account) -> None:
+    def __init__(self, account):
         self.account = account
         super(EventThread, self).__init__(name="events")
-        self.daemon = True
-        self._event_emitter = None
+        self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -205,51 +175,55 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def mark_shutdown(self) -> None:
-        lib.dc_event_emitter_close(self._event_emitter)
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
 
-    def wait(self, timeout=None) -> None:
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
         self.join(timeout=timeout)
 
-    def run(self) -> None:
+    def run(self):
         """ get and run events until shutdown. """
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
 
     def _inner_run(self):
-        if self.account._dc_context is None:
-            return
-        self._event_emitter = ffi.gc(
+        event_emitter = ffi.gc(
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while True:
-            event = lib.dc_get_next_event(self._event_emitter)
+        while not self._marked_for_shutdown:
+            event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 
             lib.dc_event_unref(event)
             ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-            for name, kwargs in self._map_ffi_event(ffi_event):
-                self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
-                hook = getattr(self.account._pm.hook, name)
-                hook(**kwargs)
+            try:
+                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
+                for name, kwargs in self._map_ffi_event(ffi_event):
+                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
+                    hook = getattr(self.account._pm.hook, name)
+                    hook(**kwargs)
+            except Exception:
+                if self.account._dc_context is not None:
+                    raise
 
-    def _map_ffi_event(self, ffi_event: FFIEvent):
+    def _map_ffi_event(self, ffi_event):
         name = ffi_event.name
         account = self.account
         if name == "DC_EVENT_CONFIGURE_PROGRESS":

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):

python/src/deltachat/message.py

@@ -3,11 +3,10 @@
 import os
 import re
 from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
+from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
 from . import const
-from datetime import datetime, timezone
-from typing import Optional
+from datetime import datetime
 
 
 class Message(object):
@@ -62,13 +61,16 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a contact request
+        If the message is a deaddrop contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.chat.accept()
-        return self.chat
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def id(self):
@@ -76,7 +78,7 @@ class Message(object):
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
-    def text(self) -> str:
+    def text(self):
         """unicode text of this messages (might be empty if not a text message). """
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
@@ -85,9 +87,9 @@ class Message(object):
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
-    def html(self) -> str:
+    def html(self):
         """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
+        return from_dc_charpointer(
             lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
@@ -114,13 +116,12 @@ class Message(object):
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
-    def basename(self) -> str:
+    def basename(self):
         """basename of the attachment if it exists, otherwise empty string. """
-        # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
     @props.with_doc
-    def filemime(self) -> str:
+    def filemime(self):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
@@ -132,7 +133,7 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
-    def get_setupcodebegin(self) -> str:
+    def get_setupcodebegin(self):
         """ return the first characters of a setup code in a setup message. """
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
@@ -140,15 +141,11 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
-    def get_message_info(self) -> str:
+    def get_message_info(self):
         """ Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
@@ -172,7 +169,7 @@ class Message(object):
         :returns: naive datetime.datetime() object.
         """
         ts = lib.dc_msg_get_timestamp(self._dc_msg)
-        return datetime.fromtimestamp(ts, timezone.utc)
+        return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def time_received(self):
@@ -182,7 +179,7 @@ class Message(object):
         """
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
+            return datetime.utcfromtimestamp(ts)
 
     @props.with_doc
     def ephemeral_timer(self):
@@ -202,14 +199,14 @@ class Message(object):
         """
         ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
         if ts:
-            return datetime.fromtimestamp(ts, timezone.utc)
+            return datetime.utcfromtimestamp(ts)
 
     @property
-    def quoted_text(self) -> Optional[str]:
+    def quoted_text(self):
         """Text inside the quote
 
         :returns: Quoted text"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
 
     @property
     def quote(self):
@@ -225,10 +222,6 @@ class Message(object):
         """Quote setter"""
         lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
 
-    def force_plaintext(self) -> None:
-        """Force the message to be sent in plain text."""
-        lib.dc_msg_force_plaintext(self._dc_msg)
-
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -246,9 +239,9 @@ class Message(object):
             return email.message_from_string(s)
 
     @property
-    def error(self) -> Optional[str]:
+    def error(self):
         """Error message"""
-        return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
+        return from_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
 
     @property
     def chat(self):
@@ -261,12 +254,12 @@ class Message(object):
         return Chat(self.account, chat_id)
 
     @props.with_doc
-    def override_sender_name(self) -> Optional[str]:
+    def override_sender_name(self):
         """the name that should be shown over the message instead of the contact display name.
 
         Usually used to impersonate someone else.
         """
-        return from_optional_dc_charpointer(
+        return from_dc_charpointer(
             lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):

python/src/deltachat/provider.py

@@ -14,7 +14,7 @@ class Provider(object):
     :param domain: The email to get the provider info for.
     """
 
-    def __init__(self, account, addr) -> None:
+    def __init__(self, account, addr):
         provider = ffi.gc(
             lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
             lib.dc_provider_unref,
@@ -24,19 +24,19 @@ class Provider(object):
         self._provider = provider
 
     @property
-    def overview_page(self) -> str:
+    def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
         return from_dc_charpointer(
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self) -> str:
+    def get_before_login_hints(self):
         """Should be shown to the user on login."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+            lib.dc_provider_get_before_login_hints(self._provider))
 
     @property
-    def status(self) -> int:
+    def status(self):
         """The status of the provider information.
 
         This is one of the

python/src/deltachat/testplugin.py

@@ -8,42 +8,34 @@ import threading
 import fnmatch
 import time
 import weakref
-from queue import Queue
-from typing import List, Callable
+import tempfile
 
 import pytest
 import requests
 
-from . import Account, const, account_hookimpl, get_core_info
+from . import Account, const
+from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
 
 def pytest_addoption(parser):
-    group = parser.getgroup("deltachat testplugin options")
-    group.addoption(
+    parser.addoption(
         "--liveconfig", action="store", default=None,
         help="a file with >=2 lines where each line "
              "contains NAME=VALUE config settings for one account"
     )
-    group.addoption(
+    parser.addoption(
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
-    group.addoption(
+    parser.addoption(
         "--strict-tls", action="store_true",
         help="Never accept invalid TLS certificates for test accounts",
     )
-    group.addoption(
-        "--extra-info", action="store_true",
-        help="show more info on failures (imap server state, config)"
-    )
-    group.addoption(
-        "--debug-setup", action="store_true",
-        help="show events during configure and start io phases of online accounts"
-    )
 
 
 def pytest_configure(config):
@@ -98,12 +90,9 @@ def pytest_configure(config):
 
         @pytest.hookimpl(hookwrapper=True)
         def pytest_runtest_teardown(self, item):
-            logging = item.config.getoption("--extra-info")
-            if logging:
-                self.enable_logging(item)
+            self.enable_logging(item)
             yield
-            if logging:
-                self.disable_logging(item)
+            self.disable_logging(item)
 
     la = LoggingAspect()
     config.pluginmanager.register(la)
@@ -111,12 +100,20 @@ def pytest_configure(config):
 
 
 def pytest_report_header(config, startdir):
-    info = get_core_info()
-    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
-        info['deltachat_core_version'],
-        info['sqlite_version'],
-        info['journal_mode'],
-    )]
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+         info['journal_mode'],
+     )])
 
     cfg = config.option.liveconfig
     if cfg:
@@ -128,60 +125,61 @@ def pytest_report_header(config, startdir):
     return summary
 
 
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
 @pytest.fixture(scope="session")
-def testprocess(request):
-    return TestProcess(pytestconfig=request.config)
-
-
-class TestProcess:
-    """ A pytest session-scoped instance to help with managing "live" account configurations.
-    """
-    def __init__(self, pytestconfig):
-        self.pytestconfig = pytestconfig
-
-    def get_liveconfig_producer(self):
-        """ provide live account configs, cached on a per-test-process scope
-        so that test functions can re-use already known live configs.
-        Depending on the --liveconfig option this comes from
-        a HTTP provider or a file with a line specifying each accounts config.
-        """
-        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
-        if not liveconfig_opt:
-            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
-
-        configlist = []
-        if not liveconfig_opt.startswith("http"):
-            for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith('#'):
-                    d = {}
-                    for part in line.split():
-                        name, value = part.split("=")
-                        d[name] = value
-                    configlist.append(d)
-
-            yield from iter(configlist)
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
         else:
-            MAX_LIVE_CREATED_ACCOUNTS = 10
-            for index in range(MAX_LIVE_CREATED_ACCOUNTS):
-                try:
-                    yield configlist[index]
-                except IndexError:
-                    res = requests.post(liveconfig_opt)
-                    if res.status_code != 200:
-                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
-                                    index, res.status_code, res.text))
-                    d = res.json()
-                    config = dict(addr=d["email"], mail_pw=d["password"])
-                    print("newtmpuser {}: addr={}".format(index, config["addr"]))
-                    configlist.append(config)
-                    yield config
-            pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS))
+            return SessionLiveConfigFromFile(liveconfig_opt)
 
 
 @pytest.fixture
 def data(request):
     class Data:
-        def __init__(self) -> None:
+        def __init__(self):
             # trying to find test data heuristically
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
@@ -208,329 +206,263 @@ def data(request):
     return Data()
 
 
-class ACSetup:
-    """ accounts setup helper to deal with multiple configure-process
-    and io & imap initialization phases. From tests, use the higher level
-    public ACFactory methods instead of its private helper class.
-    """
-    CONFIGURING = "CONFIGURING"
-    CONFIGURED = "CONFIGURED"
-    IDLEREADY = "IDLEREADY"
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
-    def __init__(self, init_time):
-        self._configured_events = Queue()
-        self._account2state = {}
-        self._imap_cleaned = set()
-        self.init_time = init_time
-
-    def start_configure(self, account, reconfigure=False):
-        """ add an account and start its configure process. """
-        class PendingTracker:
-            @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
-
-        account.add_account_plugin(PendingTracker(), name="pending_tracker")
-        self._account2state[account] = self.CONFIGURING
-        account.configure(reconfigure=reconfigure)
-        print("started configure on pending", account)
-
-    def wait_one_configured(self, account):
-        """ wait until this account has successfully configured. """
-        if self._account2state[account] == self.CONFIGURING:
-            while 1:
-                acc = self._pop_config_success()
-                if acc == account:
-                    break
-            self.init_direct_imap_and_logging(acc)
-            acc._evtracker.consume_events()
-
-    def bring_online(self):
-        """ Wait for all accounts to become ready to receive messages.
-
-        This will initialize logging, start IO and the direct_imap attribute
-        for each account which either is CONFIGURED already or which is CONFIGURING
-        and successfully completing the configuration process.
-        """
-        print("wait_all_configured finds accounts=", self._account2state)
-        for acc, state in self._account2state.items():
-            if state == self.CONFIGURED:
-                self._onconfigure_start_io(acc)
-                self._account2state[acc] = self.IDLEREADY
-
-        while self.CONFIGURING in self._account2state.values():
-            acc = self._pop_config_success()
-            self._onconfigure_start_io(acc)
-            self._account2state[acc] = self.IDLEREADY
-        print("finished, account2state", self._account2state)
-
-    def _pop_config_success(self):
-        acc, success = self._configured_events.get()
-        if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
-        self._account2state[acc] = self.CONFIGURED
-        return acc
-
-    def _onconfigure_start_io(self, acc):
-        acc.start_io()
-        print(acc._logid, "waiting for inbox IDLE to become ready")
-        acc._evtracker.wait_idle_inbox_ready()
-        self.init_direct_imap_and_logging(acc)
-        acc._evtracker.consume_events()
-        acc.log("inbox IDLE ready")
-
-    def init_direct_imap_and_logging(self, acc):
-        """ idempotent function for initializing direct_imap and logging for an account. """
-        self.init_direct_imap(acc)
-        self.init_logging(acc)
-
-    def init_logging(self, acc):
-        logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
-        acc.add_account_plugin(logger, name=acc._logid)
-
-    def init_direct_imap(self, acc):
-        """ idempotent function for initializing direct_imap."""
-        from deltachat.direct_imap import DirectImap
-        if not hasattr(acc, "direct_imap"):
-            acc.direct_imap = DirectImap(acc)
-        addr = acc.get_config("addr")
-        if addr not in self._imap_cleaned:
-            imap = acc.direct_imap
-            for folder in imap.list_folders():
-                if folder.lower() == "inbox" or folder.lower() == "deltachat":
-                    assert imap.select_folder(folder)
-                    imap.delete("1:*", expunge=True)
-                else:
-                    imap.conn.folder.delete(folder)
-            acc.log("imap cleaned for addr {}".format(addr))
-            self._imap_cleaned.add(addr)
-
-
-class ACFactory:
-    _finalizers: List[Callable[[], None]]
-    _accounts: List[Account]
-
-    def __init__(self, request, testprocess, tmpdir, data) -> None:
-        self.init_time = time.time()
-        self.tmpdir = tmpdir
-        self.pytestconfig = request.config
-        self.data = data
-        self._liveconfig_producer = testprocess.get_liveconfig_producer()
-
-        self._finalizers = []
-        self._accounts = []
-        self._acsetup = ACSetup(self.init_time)
-        self._preconfigured_keys = ["alice", "bob", "charlie",
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self._accounts = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
-        self.set_logging_default(False)
-        request.addfinalizer(self.finalize)
+            self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
-    def finalize(self):
-        while self._finalizers:
-            fin = self._finalizers.pop()
-            fin()
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
 
-        while self._accounts:
-            acc = self._accounts.pop()
-            if acc is not None:
-                imap = getattr(acc, "direct_imap", None)
-                if imap is not None:
-                    imap.shutdown()
-                    del acc.direct_imap
+            while self._accounts:
+                acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
-    def get_next_liveconfig(self):
-        """ Base function to get functional online configurations
-        where we can make valid SMTP and IMAP connections with.
-        """
-        configdict = next(self._liveconfig_producer)
-        if "e2ee_enabled" not in configdict:
-            configdict["e2ee_enabled"] = "1"
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path, logging=self._logging)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
+            ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
+            if not quiet:
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
+            self._accounts.append(ac)
+            return ac
 
-        if self.pytestconfig.getoption("--strict-tls"):
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+        def set_logging_default(self, logging):
+            self._logging = bool(logging)
 
-        assert "addr" in configdict and "mail_pw" in configdict
-        return configdict
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
-    def get_unconfigured_account(self):
-        logid = "ac{}".format(len(self._accounts) + 1)
-        path = self.tmpdir.join(logid)
-        ac = Account(path.strpath, logging=self._logging)
-        ac._logid = logid  # later instantiated FFIEventLogger needs this
-        ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-        if self.pytestconfig.getoption("--debug-setup"):
-            self._acsetup.init_logging(ac)
-        self._accounts.append(ac)
-        return ac
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
 
-    def set_logging_default(self, logging):
-        self._logging = bool(logging)
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
 
-    def remove_preconfigured_keys(self):
-        self._preconfigured_keys = []
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
 
-    def _preconfigure_key(self, account, addr):
-        # Only set a preconfigured key if we haven't used it yet for another account.
-        try:
-            keyname = self._preconfigured_keys.pop(0)
-        except IndexError:
-            pass
-        else:
-            fname_pub = self.data.read_path("key/{name}-public.asc".format(name=keyname))
-            fname_sec = self.data.read_path("key/{name}-secret.asc".format(name=keyname))
-            if fname_pub and fname_sec:
-                account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                return True
-            else:
-                print("WARN: could not use preconfigured keys for {!r}".format(addr))
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
 
-    def get_pseudo_configured_account(self):
-        # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
-        acname = os.path.basename(ac.db_path)
-        addr = "{}@offline.org".format(acname)
-        ac.update_config(dict(
-            addr=addr, displayname=acname, mail_pw="123",
-            configured_addr=addr, configured_mail_pw="123",
-            configured="1",
-        ))
-        self._preconfigure_key(ac, addr)
-        self._acsetup.init_logging(ac)
-        return ac
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
-    def new_online_configuring_account(self, cloned_from=None, **kwargs):
-        if cloned_from is None:
-            configdict = self.get_next_liveconfig()
-        else:
-            # XXX we might want to transfer the key to the new account
-            configdict = dict(
-                addr=cloned_from.get_config("addr"),
-                mail_pw=cloned_from.get_config("mail_pw"),
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac._configtracker = ac.configure()
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            self.wait_configure_and_start_io([ac1])
+            return ac1
+
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            self.wait_configure_and_start_io([ac1, ac2])
+            return ac1, ac2
+
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+                        for i in range(num)]
+            self.wait_configure_and_start_io(accounts)
+            for acc in accounts:
+                acc.add_account_plugin(FFIEventLogger(acc))
+            return accounts
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            """ Clones addr, mail_pw, mvbox_watch, mvbox_move, sentbox_watch and the
+            direct_imap object of an online account. This simulates the user setting
+            up a new device without importing a backup.
+
+            `pre_generated_key` only means that a key from python/tests/data/key is
+            used in order to speed things up.
+            """
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
+            ac._configtracker = ac.configure()
+            return ac
+
+        def wait_configure_and_start_io(self, accounts=None):
+            if accounts is None:
+                accounts = self._accounts[:]
+            started_accounts = []
+            for acc in accounts:
+                if acc not in started_accounts:
+                    self.wait_configure(acc)
+                    acc.set_config("bcc_self", "0")
+                    if acc.is_configured():
+                        acc.start_io()
+                        started_accounts.append(acc)
+                    print("{}: {} account was started".format(
+                        acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
+
+        def wait_configure(self, acc):
+            if hasattr(acc, "_configtracker"):
+                acc._configtracker.wait_finish()
+                acc._evtracker.consume_events()
+                acc.get_device_chat().mark_noticed()
+                del acc._configtracker
+
+        def run_bot_process(self, module, ffi=True):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
+            args = [
+                sys.executable,
+                "-u",
+                fn,
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            if ffi:
+                args.insert(-1, "--show-ffi")
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=0,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
             )
-        configdict.update(kwargs)
-        ac = self.prepare_account_from_liveconfig(configdict)
-        self._acsetup.start_configure(ac)
-        return ac
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
 
-    def prepare_account_from_liveconfig(self, configdict):
-        ac = self.get_unconfigured_account()
-        assert "addr" in configdict and "mail_pw" in configdict, configdict
-        configdict.setdefault("bcc_self", False)
-        configdict.setdefault("mvbox_move", False)
-        configdict.setdefault("sentbox_watch", False)
-        ac.update_config(configdict)
-        self._preconfigure_key(ac, configdict["addr"])
-        return ac
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                ac.dump_account_info(logfile=logfile)
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
 
-    def wait_configured(self, account):
-        """ Wait until the specified account has successfully completed configure. """
-        self._acsetup.wait_one_configured(account)
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
 
-    def bring_accounts_online(self):
-        print("bringing accounts online")
-        self._acsetup.bring_online()
-        print("all accounts online")
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
 
-    def get_online_accounts(self, num):
-        # to reduce number of log events logging starts after accounts can receive
-        accounts = [self.new_online_configuring_account() for i in range(num)]
-        self.bring_accounts_online()
-        return accounts
-
-    def run_bot_process(self, module, ffi=True):
-        fn = module.__file__
-
-        bot_cfg = self.get_next_liveconfig()
-        bot_ac = self.prepare_account_from_liveconfig(bot_cfg)
-
-        # Forget ac as it will be opened by the bot subprocess
-        # but keep something in the list to not confuse account generation
-        self._accounts[self._accounts.index(bot_ac)] = None
-
-        args = [
-            sys.executable,
-            "-u",
-            fn,
-            "--email", bot_cfg["addr"],
-            "--password", bot_cfg["mail_pw"],
-            bot_ac.db_path,
-        ]
-        if ffi:
-            args.insert(-1, "--show-ffi")
-        print("$", " ".join(args))
-        popen = subprocess.Popen(
-            args=args,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-            bufsize=0,                 # line buffering
-            close_fds=True,            # close all FDs other than 0/1/2
-            universal_newlines=True    # give back text
-        )
-        bot = BotProcess(popen, addr=bot_cfg["addr"])
-        self._finalizers.append(bot.kill)
-        return bot
-
-    def dump_imap_summary(self, logfile):
-        for ac in self._accounts:
-            ac.dump_account_info(logfile=logfile)
-            imap = getattr(ac, "direct_imap", None)
-            if imap is not None:
-                try:
-                    imap.idle_done()
-                except Exception:
-                    pass
-                imap.dump_imap_structures(self.tmpdir, logfile=logfile)
-
-    def get_accepted_chat(self, ac1: Account, ac2: Account):
-        ac2.create_chat(ac1)
-        return ac1.create_chat(ac2)
-
-    def introduce_each_other(self, accounts, sending=True):
-        to_wait = []
-        for i, acc in enumerate(accounts):
-            for acc2 in accounts[i + 1:]:
-                chat = self.get_accepted_chat(acc, acc2)
-                if sending:
-                    chat.send_text("hi")
-                    to_wait.append(acc2)
-                    acc2.create_chat(acc).send_text("hi back")
-                    to_wait.append(acc)
-        for acc in to_wait:
-            acc._evtracker.wait_next_incoming_message()
-
-
-@pytest.fixture
-def acfactory(request, tmpdir, testprocess, data):
-    am = ACFactory(request=request, tmpdir=tmpdir, testprocess=testprocess, data=data)
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
     yield am
     if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        if testprocess.pytestconfig.getoption("--extra-info"):
-            logfile = io.StringIO()
-            am.dump_imap_summary(logfile=logfile)
-            print(logfile.getvalue())
-            # request.node.add_report_section("call", "imap-server-state", s)
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
-    stdout_queue: queue.Queue
-
-    def __init__(self, popen, addr) -> None:
+    def __init__(self, popen, bot_cfg):
         self.popen = popen
-        self.addr = addr
+        self.addr = bot_cfg["addr"]
 
         # we read stdout as quickly as we can in a thread and make
         # the (unicode) lines available for readers through a queue.
         self.stdout_queue = queue.Queue()
         self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.daemon = True
+        t.setDaemon(1)
         t.start()
 
-    def _run_stdout_thread(self) -> None:
+    def _run_stdout_thread(self):
         try:
             while 1:
                 line = self.popen.stdout.readline()
@@ -542,10 +474,10 @@ class BotProcess:
         finally:
             self.stdout_queue.put(None)
 
-    def kill(self) -> None:
+    def kill(self):
         self.popen.kill()
 
-    def wait(self, timeout=None) -> None:
+    def wait(self, timeout=30):
         self.popen.wait(timeout=timeout)
 
     def fnmatch_lines(self, pattern_lines):
@@ -554,7 +486,7 @@ class BotProcess:
             print("+++FNMATCH:", next_pattern)
             ignored = []
             while 1:
-                line = self.stdout_queue.get()
+                line = self.stdout_queue.get(timeout=15)
                 if line is None:
                     if ignored:
                         print("BOT stdout terminated after these lines")
@@ -577,14 +509,14 @@ def tmp_db_path(tmpdir):
 @pytest.fixture
 def lp():
     class Printer:
-        def sec(self, msg: str) -> None:
+        def sec(self, msg):
             print()
             print("=" * 10, msg, "=" * 10)
 
-        def step(self, msg: str) -> None:
+        def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
 
-        def indent(self, msg: str) -> None:
+        def indent(self, msg):
             print("  " + msg)
 
     return Printer()

python/src/deltachat/tracker.py

@@ -90,11 +90,11 @@ class ConfigureTracker:
             if data1 is None or evdata == data1:
                 break
 
-    def wait_finish(self, timeout=None):
+    def wait_finish(self):
         """ wait until configure is completed.
 
         Raise Exception if Configure failed
         """
-        if not self._configure_events.get(timeout=timeout):
+        if not self._configure_events.get():
             content = "\n".join(map(str, self._ffi_events))
             raise ConfigureFailed(content)

python/tests/auditwheels.py

@@ -1,17 +1,15 @@
 
 import os
-import platform
-import subprocess
 import sys
+import subprocess
 
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
                 ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                 "--plat", "manylinux2014_x86_64"])

python/tests/bench_test_setup.py

@@ -1,18 +0,0 @@
-"""
-run with:
-
-     pytest -vv --durations=10 bench_empty.py
-
-to see timings of test setups.
-"""
-
-import pytest
-
-
-class TestEmpty:
-    def test_prepare_setup_measurings(self, acfactory):
-        acfactory.get_online_accounts(5)
-
-    @pytest.mark.parametrize("num", range(0, 5))
-    def test_setup_online_accounts(self, acfactory, num):
-        acfactory.get_online_accounts(num)

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3)
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account

python/tests/test_increation.py

@@ -6,6 +6,8 @@ import shutil
 import pytest
 from filecmp import cmp
 
+from deltachat import const
+
 
 def wait_msg_delivered(account, msg_list):
     """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
@@ -33,7 +35,7 @@ def wait_msgs_changed(account, msgs_list):
 
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
@@ -43,7 +45,7 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
         chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
@@ -56,7 +58,7 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
+        ac1, ac2 = acfactory.get_two_online_accounts()
 
         chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
@@ -100,10 +102,14 @@ class TestOnlineInCreation:
         ])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -6,47 +6,9 @@ from deltachat import register_global_plugin
 from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
-from deltachat.testplugin import ACSetup
 # from deltachat.account import EventLogger
 
 
-class TestACSetup:
-    def test_basic_states(self, acfactory, monkeypatch):
-        pc = ACSetup(init_time=0.0)
-        acc = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
-        pc.start_configure(acc)
-        assert pc._account2state[acc] == pc.CONFIGURING
-        pc._configured_events.put((acc, True))
-        monkeypatch.setattr(pc, "init_direct_imap", lambda *args, **kwargs: None)
-        pc.wait_one_configured(acc)
-        assert pc._account2state[acc] == pc.CONFIGURED
-        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
-        pc.bring_online()
-        assert pc._account2state[acc] == pc.IDLEREADY
-
-    def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory):
-        pc = ACSetup(init_time=0.0)
-        monkeypatch.setattr(pc, "init_direct_imap", lambda *args, **kwargs: None)
-        monkeypatch.setattr(pc, "_onconfigure_start_io", lambda *args, **kwargs: None)
-        ac1 = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(ac1, "configure", lambda **kwargs: None)
-        pc.start_configure(ac1)
-        ac2 = acfactory.get_unconfigured_account()
-        monkeypatch.setattr(ac2, "configure", lambda **kwargs: None)
-        pc.start_configure(ac2)
-        assert pc._account2state[ac1] == pc.CONFIGURING
-        assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac1, True))
-        pc.wait_one_configured(ac1)
-        assert pc._account2state[ac1] == pc.CONFIGURED
-        assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac2, True))
-        pc.bring_online()
-        assert pc._account2state[ac1] == pc.IDLEREADY
-        assert pc._account2state[ac2] == pc.IDLEREADY
-
-
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
     capi.lib.dc_context_unref(ctx)
@@ -74,8 +36,7 @@ def test_wrong_db(tmpdir):
     # write an invalid database file
     p.write("x123" * 10)
 
-    context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
-    assert not lib.dc_context_is_open(context)
+    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):
@@ -106,7 +67,8 @@ def test_sig():
 
 
 def test_markseen_invalid_message_ids(acfactory):
-    ac1 = acfactory.get_pseudo_configured_account()
+    ac1 = acfactory.get_configured_offline_account()
+
     contact1 = ac1.create_contact("some1@example.com", name="some1")
     chat = contact1.create_chat()
     chat.send_text("one messae")
@@ -117,7 +79,7 @@ def test_markseen_invalid_message_ids(acfactory):
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):
-    ac1 = acfactory.get_pseudo_configured_account()
+    ac1 = acfactory.get_configured_offline_account()
     for i in range(1, 10):
         msg = ac1.get_message_by_id(i)
         assert msg.id == 0

python/tox.ini

@@ -1,14 +1,13 @@
 [tox]
-isolated_build = true
+# make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py3
+    py37
     lint
-    mypy
     auditwheels
 
 [testenv]
 commands = 
-    pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -44,19 +43,12 @@ commands =
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
-[testenv:mypy]
-deps =
-    mypy
-    typing
-    types-setuptools
-    types-requests
-commands =
-    mypy --no-incremental src/
-
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    # With Python 3.7 and Sphinx 3.5.0, it throws an exception.
+    # Pin the version to the working one.
+    sphinx==3.4.3
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -78,8 +70,8 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 150
-timeout_func_only = True
+timeout = 90
+timeout_method = thread
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.
 

rust-toolchain

@@ -1 +1 @@
-1.60.0
+1.50.0

scripts/README.md

@@ -1,13 +1,16 @@
 # Continuous Integration Scripts for Delta Chat
 
 Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Actions](https://docs.github.com/actions),
+[CircleCI](https://app.circleci.com/) and an own build machine.
 
 ## Description of scripts 
 
 - `../.github/workflows` contains jobs run by GitHub Actions.
 
+- `../.circleci/config.yml` describing the build jobs that are run
+  by CircleCI.
+
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
 
@@ -45,7 +48,3 @@ to avoid the relatively large download::
     cd scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-Additionally, you can install qemu and build arm64 docker image:
-    apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64

scripts/ci_upload.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
+
+
+# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
+export BRANCH=${CIRCLE_BRANCH:master}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
+
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
+
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

scripts/concourse/README.md

@@ -1,21 +0,0 @@
-# Concourse CI pipeline
-
-`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
-that builds C documentation, Python documentation, Python wheels for `x86_64`
-and `aarch64` and Python source packages, and uploads them.
-
-To setup the pipeline run
-```
-fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
-```
-where `secret.yml` contains the following secrets:
-```
-c.delta.chat:
-  private_key: |
-    -----BEGIN RSA PRIVATE KEY-----
-    ...
-    -----END RSA PRIVATE KEY-----
-devpi:
-  login: dc
-  password: ...
-```

scripts/concourse/docs_wheels.yml

@@ -1,232 +0,0 @@
-resources:
-- name: deltachat-core-rust
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-
-- name: deltachat-core-rust-release
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-    tag_filter: "py-*"
-
-jobs:
-- name: doxygen
-  plan:
-  - get: deltachat-core-rust
-    trigger: true
-
-  # Build Doxygen documentation
-  - task: build-doxygen
-    config:
-      inputs:
-        - name: deltachat-core-rust
-      outputs:
-        - name: c-docs
-      image_resource:
-        source:
-          repository: hrektts/doxygen
-        type: registry-image
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
-          cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
-
-  - task: upload-c-docs
-    config:
-      inputs:
-        - name: c-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
-- name: python-x86_64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-docs
-          path: ./python/doc/_build/
-        # Source packages
-        - name: py-dist
-          path: ./python/.docker-tox/dist/
-        # Binary wheels
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload python docs to py.delta.chat
-  - task: upload-py-docs
-    config:
-      inputs:
-        - name: py-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
-  # Upload x86_64 wheels and source packages
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-        - name: py-dist
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*
-          devpi upload py-dist/*
-
-- name: python-aarch64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload aarch64 wheels
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*

scripts/coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-if ! command -v grcov >/dev/null; then
+if ! which grcov 2>/dev/null 1>&2; then
     echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
     echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
     exit 1

scripts/docker-coredeps-arm64/Dockerfile

@@ -1,21 +0,0 @@
-FROM quay.io/pypa/manylinux2014_aarch64
-
-# Configure ld.so/ldconfig and pkg-config
-RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
-    echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
-ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
-
-# Install a recent Perl, needed to install the openssl crate
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

scripts/docker-coredeps-arm64/deps/build_openssl.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-OPENSSL_VERSION=1.1.1a
-OPENSSL_SHA256=fc20130f8b7cbd2fb918b2f14e2f429e109c31ddd0fb38fc5d71d9ffed3f9f41
-
-curl -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz
-echo "${OPENSSL_SHA256}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf openssl-${OPENSSL_VERSION}.tar.gz
-cd openssl-${OPENSSL_VERSION} 
-./config shared no-ssl2 no-ssl3 -fPIC --prefix=/usr/local
-
-sed -i "s/^SHLIB_MAJOR=.*/SHLIB_MAJOR=200/" Makefile && \
-sed -i "s/^SHLIB_MINOR=.*/SHLIB_MINOR=0.0/" Makefile && \
-sed -i "s/^SHLIB_VERSION_NUMBER=.*/SHLIB_VERSION_NUMBER=200.0.0/" Makefile
-
-make depend
-make 
-make install_sw install_ssldirs
-ldconfig -v | grep ssl 

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
-curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar -xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION}
-
-./Configure -de
-make
-make install

scripts/docker-coredeps-arm64/deps/build_python.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -x -e
-
-# we use the python3.7 environment as the base environment 
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel 
-
-pushd /usr/bin
-
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
-
-popd

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-set -e -x
-
-# Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.60.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
-rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/Dockerfile

@@ -1,11 +1,11 @@
-FROM quay.io/pypa/manylinux2014_x86_64
+FROM quay.io/pypa/manylinux2010_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1

scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

scripts/docker-coredeps/deps/build_python.sh

@@ -2,13 +2,13 @@
 
 set -x -e
 
-# we use the python3.7 environment as the base environment
-/opt/python/cp37-cp37m/bin/pip install tox devpi-client auditwheel
+# we use the python3.5 environment as the base environment 
+/opt/python/cp35-cp35m/bin/pip install tox devpi-client auditwheel 
 
 pushd /usr/bin
 
-ln -s /opt/_internal/cpython-3.7.*/bin/tox
-ln -s /opt/_internal/cpython-3.7.*/bin/devpi
-ln -s /opt/_internal/cpython-3.7.*/bin/auditwheel
+ln -s /opt/_internal/cpython-3.5.*/bin/tox
+ln -s /opt/_internal/cpython-3.5.*/bin/devpi
+ln -s /opt/_internal/cpython-3.5.*/bin/auditwheel
 
 popd

scripts/docker-coredeps/deps/build_rust.sh

@@ -3,16 +3,9 @@
 set -e -x
 
 # Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.60.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.50.0-x86_64-unknown-linux-gnu -y
+export PATH=/root/.cargo/bin:$PATH
 rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/1.50.0-x86_64-unknown-linux-gnu/share

scripts/manual_remote_tests.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
+
 set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
 
-JOB=${1:?need to specify 'rust' or 'python'}
-BRANCH="$(git branch | grep \* | cut -d ' ' -f2)"
-REPONAME="$(basename $(git rev-parse --show-toplevel))"
-
-time bash "scripts/remote_tests_$JOB.sh" "$USER-$BRANCH-$REPONAME"
+time bash scripts/$CIRCLE_JOB.sh 

scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

scripts/old/run-python.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

scripts/remote_python_packaging.sh

@@ -0,0 +1,49 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    shopt -s huponexit
+    cd $BUILDDIR
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

scripts/remote_tests_python.sh

@@ -1,9 +1,10 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
@@ -17,7 +18,7 @@ rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
 set +x
 
-echo "--- Running Python tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e

scripts/remote_tests_rust.sh

@@ -1,19 +1,20 @@
 #!/bin/bash 
 
-BUILD_ID=${1:?specify build ID}
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-BUILDDIR=ci_builds/$BUILD_ID
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
 
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
 rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
 
-echo "--- Running Rust tests remotely"
+echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e

scripts/run_all.sh

@@ -19,15 +19,14 @@ export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
 export PYTHONDONTWRITEBYTECODE=1
 pushd /bin
-rm -f python3.7
+rm -f python3.5
+ln -s /opt/python/cp35-cp35m/bin/python3.5
+ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
-rm -f python3.8
 ln -s /opt/python/cp38-cp38/bin/python3.8
-rm -f python3.9
-ln -s /opt/python/cp39-cp39/bin/python3.9
 popd
 
 pushd python
@@ -41,7 +40,7 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,auditwheels
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 
 

scripts/set_core_version.py

@@ -82,12 +82,9 @@ def main():
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
-    print("after commit, on master make sure to: ")
+    print("after commit make sure to: ")
     print("")
-    print("   git tag -a {}".format(newversion))
-    print("   git push origin {}".format(newversion))
-    print("   git tag -a py-{}".format(newversion))
-    print("   git push origin py-{}".format(newversion))
+    print("   git tag {}".format(newversion))
     print("")
 
 

spec.md

@@ -1,6 +1,6 @@
 # chat-mail specification
 
-Version: 0.33.0
+Version: 0.32.0
 Status:  In-progress 
 Format:  [Semantic Line Breaks](https://sembr.org/)
 
@@ -301,9 +301,9 @@ to add a `Chat-Group-Avatar` only on image changes.
 
 A user MAY have a profile-image that MAY be distributed to their contacts.
 To change or set the profile-image,
-the messenger MUST add the header `Chat-User-Avatar: base64:IMAGEDATA`.
-To bypass limits of headers, it is recommended not to use the outer header
-and to limit the size to 20k.
+the messenger MUST attach an image file to a message
+and MUST add the header `Chat-User-Avatar`
+with the value set to the image name.
 
 To remove the profile-image,
 the messenger MUST add the header `Chat-User-Avatar: 0`.
@@ -320,14 +320,19 @@ The messenger SHOULD NOT send an explicit mail to normal MUAs.
     From: sender@domain
     To: rcpt@domain
     Chat-Version: 1.0
+    Chat-User-Avatar: photo.jpg
     Subject: Chat: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 
     --==break==
     Content-Type: text/plain
-    Chat-User-Avatar: base64:AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQY ...
 
     Hello, I've changed my profile image.
+    --==break==
+    Content-Type: image/jpeg
+    Content-Disposition: attachment; filename="photo.jpg"
+
+    AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQYFBA ...
     --==break==--
 
 The image format SHOULD be image/jpeg or image/png.
@@ -337,11 +342,6 @@ in the same message.
 To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
-In older specs, the profile-image was sent as an attachment
-and `Chat-User-Avatar:` specified its name.
-However, it turned out that these attachements are kind of unuexpected to users,
-therefore the profile-image go to the header now.
-
 
 # Locations
 
@@ -401,41 +401,9 @@ it is fine if the location is detected on forwarding etc.
     </kml>
 
 
-# Stickers
+# Miscellaneous
 
-Stickers are send as normal images
-with the additional header `Chat-Content: sticker`.
-
-It is discouraged to send stickers together with user generated text,
-however, stickers can be used as a reply to a message
-and also the footer should be set as usual.
-
-    From: alice@example.org
-    To: bob@example.com
-    Chat-Version: 1.0
-    Chat-Content: sticker
-    Message-ID: Mr.12345uvwxyZ.0005@example.org
-    Subject: Message from Alice
-    Content-Type: multipart/mixed; boundary="==break=="
-
-    --==break==
-    Content-Type: text/plain
-
-    -- 
-    Hi there! I am using this new messenger!
-    --==break==
-    Content-Type: image/png
-    Content-Disposition: attachment; filename="sticker.png"
-
-    R0lGODlhpAGkAfe9AP+zd2eQkZhrI//z9v++PMb///+scrdDT3BtbtrZ2f/LQSsREcdIVf9 ...
-    --==break==--
-
-Typical sticker formats are `image/png`, `image/gif` and `image/webp`.
-Animated stickers are supported
-by just using an image format that supports animation.
-
-
-# Voice messages
+Messengers SHOULD use the header `In-Reply-To` as usual.
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -449,11 +417,6 @@ This allows the receiver to show the time without knowing the file format.
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 
-
-# Miscellaneous
-
-Messengers SHOULD use the header `In-Reply-To` as usual.
-
 Messengers MAY send and receive Message Disposition Notifications
 (MDNs, [RFC 8098](https://tools.ietf.org/html/rfc8098),
 [RFC 3503](https://tools.ietf.org/html/rfc3503))
@@ -474,4 +437,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2021 Delta Chat contributors.
+Copyright © 2017-2020 Delta Chat contributors.

src/accounts.rs

@@ -1,8 +1,5 @@
-//! # Account manager module.
-
 use std::collections::BTreeMap;
 
-use async_std::channel::{self, Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
@@ -13,37 +10,39 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 
 use crate::context::Context;
-use crate::events::{Event, EventType, Events};
+use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: BTreeMap<u32, Context>,
-    emitter: EventEmitter,
-
-    /// Event channel to emit account manager errors.
-    events: Events,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
 }
 
 impl Accounts {
     /// Loads or creates an accounts folder at the given `dir`.
-    pub async fn new(dir: PathBuf) -> Result<Self> {
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
         if !dir.exists().await {
-            Accounts::create(&dir).await?;
+            Accounts::create(os_name, &dir).await?;
         }
 
         Accounts::open(dir).await
     }
 
-    /// Creates a new default structure.
-    pub async fn create(dir: &PathBuf) -> Result<()> {
+    /// Creates a new default structure, including a default account.
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(dir).await?;
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
 
         Ok(())
     }
@@ -54,121 +53,64 @@ impl Accounts {
         ensure!(dir.exists().await, "directory does not exist");
 
         let config_file = dir.join(CONFIG_NAME);
-        ensure!(
-            config_file.exists().await,
-            "{:?} does not exist",
-            config_file
-        );
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
 
-        let config = Config::from_file(config_file)
-            .await
-            .context("failed to load accounts config")?;
-        let accounts = config
-            .load_accounts()
-            .await
-            .context("failed to load accounts")?;
-
-        let emitter = EventEmitter::new();
-
-        let events = Events::default();
-
-        emitter.sender.send(events.get_emitter()).await?;
-
-        for account in accounts.values() {
-            emitter.add_account(account).await.with_context(|| {
-                format!("failed to add account {} to event emitter", account.id)
-            })?;
-        }
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
 
         Ok(Self {
             dir,
             config,
-            accounts,
-            emitter,
-            events,
+            accounts: Arc::new(RwLock::new(accounts)),
         })
     }
 
     /// Get an account by its `id`:
     pub async fn get_account(&self, id: u32) -> Option<Context> {
-        self.accounts.get(&id).cloned()
+        self.accounts.read().await.get(&id).cloned()
     }
 
     /// Get the currently selected account.
-    pub async fn get_selected_account(&self) -> Option<Context> {
+    pub async fn get_selected_account(&self) -> Context {
         let id = self.config.get_selected_account().await;
-        self.accounts.get(&id).cloned()
-    }
-
-    /// Returns the currently selected account's id or None if no account is selected.
-    pub async fn get_selected_account_id(&self) -> Option<u32> {
-        match self.config.get_selected_account().await {
-            0 => None,
-            id => Some(id),
-        }
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
     }
 
     /// Select the given account.
-    pub async fn select_account(&mut self, id: u32) -> Result<()> {
+    pub async fn select_account(&self, id: u32) -> Result<()> {
         self.config.select_account(id).await?;
 
         Ok(())
     }
 
-    /// Add a new account and opens it.
-    ///
-    /// Returns account ID.
-    pub async fn add_account(&mut self) -> Result<u32> {
+    /// Add a new account.
+    pub async fn add_account(&self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
         let account_config = self.config.new_account(&self.dir).await?;
 
-        let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
-        self.accounts.insert(account_config.id, ctx);
-
-        Ok(account_config.id)
-    }
-
-    /// Adds a new closed account.
-    pub async fn add_closed_account(&mut self) -> Result<u32> {
-        let account_config = self.config.new_account(&self.dir).await?;
-
-        let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
-        self.accounts.insert(account_config.id, ctx);
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.accounts.write().await.insert(account_config.id, ctx);
 
         Ok(account_config.id)
     }
 
     /// Remove an account.
-    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
-        let ctx = self.accounts.remove(&id);
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
         ensure!(ctx.is_some(), "no account with this id: {}", id);
         let ctx = ctx.unwrap();
         ctx.stop_io().await;
         drop(ctx);
 
         if let Some(cfg) = self.config.get_account(id).await {
-            // Spend up to 1 minute trying to remove the files.
-            // Files may remain locked up to 30 seconds due to r2d2 bug:
-            // https://github.com/sfackler/r2d2/issues/99
-            let mut counter = 0;
-            loop {
-                counter += 1;
-
-                if let Err(err) = fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
-                    .await
-                    .context("failed to remove account data")
-                {
-                    if counter > 60 {
-                        return Err(err);
-                    }
-
-                    // Wait 1 second and try again.
-                    async_std::task::sleep(std::time::Duration::from_millis(1000)).await;
-                } else {
-                    break;
-                }
-            }
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
         }
         self.config.remove_account(id).await?;
 
@@ -176,9 +118,8 @@ impl Accounts {
     }
 
     /// Migrate an existing account into this structure.
-    pub async fn migrate_account(&mut self, dbfile: PathBuf) -> Result<u32> {
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
-        let walfile = Context::derive_walfile(&dbfile);
 
         ensure!(
             dbfile.exists().await,
@@ -202,7 +143,6 @@ impl Accounts {
 
         let new_dbfile = account_config.dbfile().into();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
-        let new_walfile = Context::derive_walfile(&new_dbfile);
 
         let res = {
             fs::create_dir_all(&account_config.dir)
@@ -214,19 +154,19 @@ impl Accounts {
             fs::rename(&blobdir, &new_blobdir)
                 .await
                 .context("failed to rename blobdir")?;
-            if walfile.exists().await {
-                fs::rename(&walfile, &new_walfile)
-                    .await
-                    .context("failed to rename walfile")?;
-            }
             Ok(())
         };
 
         match res {
             Ok(_) => {
-                let ctx = Context::new(new_dbfile, account_config.id).await?;
-                self.emitter.add_account(&ctx).await?;
-                self.accounts.insert(account_config.id, ctx);
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
+                self.accounts.write().await.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
@@ -247,114 +187,75 @@ impl Accounts {
 
     /// Get a list of all account ids.
     pub async fn get_all(&self) -> Vec<u32> {
-        self.accounts.keys().copied().collect()
+        self.accounts.read().await.keys().copied().collect()
     }
 
-    /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
-    ///
-    /// Returns whether all accounts finished their background work.
-    /// DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
-    ///
-    /// iOS can:
-    /// - call dc_start_io() (in case IO was not running)
-    /// - call dc_maybe_network()
-    /// - while dc_accounts_all_work_done() returns false:
-    ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
-    pub async fn all_work_done(&self) -> bool {
-        for account in self.accounts.values() {
-            if !account.all_work_done().await {
-                return false;
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
             }
         }
-        true
     }
 
     pub async fn start_io(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.start_io().await;
         }
     }
 
     pub async fn stop_io(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.stop_io().await;
         }
     }
 
     pub async fn maybe_network(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.maybe_network().await;
         }
     }
 
-    pub async fn maybe_network_lost(&self) {
-        for account in self.accounts.values() {
-            account.maybe_network_lost().await;
-        }
-    }
-
-    /// Emits a single event.
-    pub fn emit_event(&self, event: EventType) {
-        self.events.emit(Event { id: 0, typ: event })
-    }
-
-    /// Returns unified event emitter.
+    /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        self.emitter.clone()
+        let emitters: Vec<_> = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(_id, a)| a.get_event_emitter())
+            .collect();
+
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
-/// Unified event emitter for multiple accounts.
-#[derive(Debug, Clone)]
-pub struct EventEmitter {
-    /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<crate::events::EventEmitter>>>,
-
-    /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<crate::events::EventEmitter>,
-
-    /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<crate::events::EventEmitter>,
-}
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
 
 impl EventEmitter {
-    pub fn new() -> Self {
-        let (sender, receiver) = channel::unbounded();
-        Self {
-            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
-            sender,
-            receiver,
-        }
-    }
-
     /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
     pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv()).unwrap_or_default()
+        async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
-    pub async fn recv(&mut self) -> Result<Option<Event>> {
-        let mut stream = self.stream.write().await;
-        loop {
-            match futures::future::select(self.receiver.recv(), stream.next()).await {
-                futures::future::Either::Left((emitter, _)) => {
-                    stream.push(emitter?);
-                }
-                futures::future::Either::Right((ev, _)) => return Ok(ev),
-            }
-        }
-    }
-
-    /// Add event emitter of a new account to the aggregate event emitter.
-    pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender.send(context.get_event_emitter()).await?;
-        Ok(())
-    }
-}
-
-impl Default for EventEmitter {
-    fn default() -> Self {
-        Self::new()
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
@@ -365,25 +266,22 @@ impl async_std::stream::Stream for EventEmitter {
         mut self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self).poll_next(cx)
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
     }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
-/// Account manager configuration file.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone)]
 pub struct Config {
     file: PathBuf,
-    inner: InnerConfig,
+    inner: Arc<RwLock<InnerConfig>>,
 }
 
-/// Account manager configuration file contents.
-///
-/// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
+    pub os_name: String,
     /// The currently selected account.
     pub selected_account: u32,
     pub next_id: u32,
@@ -391,15 +289,15 @@ struct InnerConfig {
 }
 
 impl Config {
-    pub async fn new(dir: &PathBuf) -> Result<Self> {
-        let inner = InnerConfig {
-            accounts: Vec::new(),
-            selected_account: 0,
-            next_id: 1,
-        };
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
         let cfg = Config {
             file: dir.join(CONFIG_NAME),
-            inner,
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
         };
 
         cfg.sync().await?;
@@ -407,11 +305,18 @@ impl Config {
         Ok(cfg)
     }
 
+    pub async fn os_name(&self) -> String {
+        self.inner.read().await.os_name.clone()
+    }
+
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
-        fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
-            .await
-            .context("failed to write config")
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
     }
 
     /// Read a configuration from the given file into memory.
@@ -419,21 +324,22 @@ impl Config {
         let bytes = fs::read(&file).await.context("failed to read file")?;
         let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
 
-        Ok(Config { file, inner })
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
     }
 
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
         let mut accounts = BTreeMap::new();
-        for account_config in &self.inner.accounts {
-            let ctx = Context::new(account_config.dbfile().into(), account_config.id)
-                .await
-                .with_context(|| {
-                    format!(
-                        "failed to create context from file {:?}",
-                        account_config.dbfile()
-                    )
-                })?;
-
+        for account_config in &cfg.accounts {
+            let ctx = Context::new(
+                cfg.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
             accounts.insert(account_config.id, ctx);
         }
 
@@ -441,67 +347,70 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
-            let id = self.inner.next_id;
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
             let uuid = Uuid::new_v4();
             let target_dir = dir.join(uuid.to_simple_ref().to_string());
 
-            self.inner.accounts.push(AccountConfig {
+            inner.accounts.push(AccountConfig {
                 id,
                 dir: target_dir.into(),
                 uuid,
             });
-            self.inner.next_id += 1;
+            inner.next_id += 1;
             id
         };
 
         self.sync().await?;
 
-        self.select_account(id)
-            .await
-            .context("failed to select just added account")?;
-        let cfg = self
-            .get_account(id)
-            .await
-            .context("failed to get just added account")?;
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
         Ok(cfg)
     }
 
     /// Removes an existing acccount entirely.
-    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
         {
-            if let Some(idx) = self.inner.accounts.iter().position(|e| e.id == id) {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
                 // remove account from the configs
-                self.inner.accounts.remove(idx);
+                inner.accounts.remove(idx);
             }
-            if self.inner.selected_account == id {
+            if inner.selected_account == id {
                 // reset selected account
-                self.inner.selected_account =
-                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
             }
         }
 
         self.sync().await
     }
 
-    async fn get_account(&self, id: u32) -> Option<AccountConfig> {
-        self.inner.accounts.iter().find(|e| e.id == id).cloned()
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
     }
 
     pub async fn get_selected_account(&self) -> u32 {
-        self.inner.selected_account
+        self.inner.read().await.selected_account
     }
 
-    pub async fn select_account(&mut self, id: u32) -> Result<()> {
+    pub async fn select_account(&self, id: u32) -> Result<()> {
         {
+            let inner = &mut *self.inner.write().await;
             ensure!(
-                self.inner.accounts.iter().any(|e| e.id == id),
+                inner.accounts.iter().any(|e| e.id == id),
                 "invalid account id: {}",
                 id
             );
 
-            self.inner.selected_account = id;
+            inner.selected_account = id;
         }
 
         self.sync().await?;
@@ -509,9 +418,8 @@ impl Config {
     }
 }
 
-/// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-struct AccountConfig {
+pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
     /// Root directory for all data for this account.
@@ -535,17 +443,21 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let mut accounts1 = Accounts::new(p.clone()).await.unwrap();
-        accounts1.add_account().await.unwrap();
-
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         let accounts2 = Accounts::open(p).await.unwrap();
 
-        assert_eq!(accounts1.accounts.len(), 1);
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
         assert_eq!(accounts1.config.get_selected_account().await, 1);
 
         assert_eq!(accounts1.dir, accounts2.dir);
-        assert_eq!(accounts1.config, accounts2.config,);
-        assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
     }
 
     #[async_std::test]
@@ -553,47 +465,22 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.len(), 1);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 2);
         assert_eq!(accounts.config.get_selected_account().await, id);
-        assert_eq!(accounts.accounts.len(), 2);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
 
         accounts.select_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         accounts.remove_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.len(), 1);
-    }
-
-    #[async_std::test]
-    async fn test_accounts_remove_last() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new(p.clone()).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-        assert_eq!(accounts.config.get_selected_account().await, 0);
-
-        let id = accounts.add_account().await?;
-        assert!(accounts.get_selected_account().await.is_some());
-        assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, id);
-
-        accounts.remove_account(id).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-
-        Ok(())
+        assert_eq!(accounts.accounts.read().await.len(), 1);
     }
 
     #[async_std::test]
@@ -601,12 +488,14 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
-        let ctx = Context::new(extern_dbfile.clone(), 0).await.unwrap();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
         ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
             .await
             .unwrap();
@@ -617,10 +506,10 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
 
-        let ctx = accounts.get_selected_account().await.unwrap();
+        let ctx = accounts.get_selected_account().await;
         assert_eq!(
             "me@mail.com",
             ctx.get_config(crate::config::Config::Addr)
@@ -636,9 +525,9 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new(p.clone()).await.unwrap();
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        for expected_id in 1..10 {
+        for expected_id in 2..10 {
             let id = accounts.add_account().await.unwrap();
             assert_eq!(id, expected_id);
         }
@@ -648,157 +537,4 @@ mod tests {
             assert_eq!(ids.get(i), Some(&expected_id));
         }
     }
-
-    #[async_std::test]
-    async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-        let dummy_accounts = 10;
-
-        let (id0, id1, id2) = {
-            let mut accounts = Accounts::new(p.clone()).await?;
-            accounts.add_account().await?;
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 1);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("one@example.org"))
-                .await?;
-
-            let id1 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id1).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("two@example.org"))
-                .await?;
-
-            // add and remove some accounts and force a gap (ids must not be reused)
-            for _ in 0..dummy_accounts {
-                let to_delete = accounts.add_account().await?;
-                accounts.remove_account(to_delete).await?;
-            }
-
-            let id2 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id2).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("three@example.org"))
-                .await?;
-
-            accounts.select_account(id1).await?;
-
-            (id0, id1, id2)
-        };
-        assert!(id0 > 0);
-        assert!(id1 > id0);
-        assert!(id2 > id1 + dummy_accounts);
-
-        let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new(p.clone()).await?;
-            let ctx = accounts.get_selected_account().await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 3);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("one@example.org".to_string())
-            );
-
-            let id1 = *ids.get(1).unwrap();
-            let t = accounts.get_account(id1).await.unwrap();
-            assert_eq!(
-                t.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let id2 = *ids.get(2).unwrap();
-            let ctx = accounts.get_account(id2).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("three@example.org".to_string())
-            );
-
-            (id0, id1, id2)
-        };
-        assert_eq!(id0, id0_reopened);
-        assert_eq!(id1, id1_reopened);
-        assert_eq!(id2, id2_reopened);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_no_accounts_event_emitter() -> Result<()> {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new(p.clone()).await?;
-
-        // Make sure there are no accounts.
-        assert_eq!(accounts.accounts.len(), 0);
-
-        // Create event emitter.
-        let mut event_emitter = accounts.get_event_emitter().await;
-
-        // Test that event emitter does not return `None` immediately.
-        let duration = std::time::Duration::from_millis(1);
-        assert!(async_std::future::timeout(duration, event_emitter.recv())
-            .await
-            .is_err());
-
-        // When account manager is dropped, event emitter is exhausted.
-        drop(accounts);
-        assert_eq!(event_emitter.recv().await?, None);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_encrypted_account() -> Result<()> {
-        let dir = tempfile::tempdir().context("failed to create tempdir")?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create accounts manager")?;
-
-        assert_eq!(accounts.accounts.len(), 0);
-        let account_id = accounts
-            .add_closed_account()
-            .await
-            .context("failed to add closed account")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.id, account_id);
-        let passphrase_set_success = account
-            .open("foobar".to_string())
-            .await
-            .context("failed to set passphrase")?;
-        assert!(passphrase_set_success);
-        drop(accounts);
-
-        let accounts = Accounts::new(p.clone())
-            .await
-            .context("failed to create second accounts manager")?;
-        let account = accounts
-            .get_selected_account()
-            .await
-            .context("failed to get account")?;
-        assert_eq!(account.is_open().await, false);
-
-        // Try wrong passphrase.
-        assert_eq!(account.open("barfoo".to_string()).await?, false);
-        assert_eq!(account.open("".to_string()).await?, false);
-
-        assert_eq!(account.open("foobar".to_string()).await?, true);
-        assert_eq!(account.is_open().await, true);
-
-        Ok(())
-    }
 }

src/aheader.rs

@@ -1,13 +1,13 @@
-//! # Autocrypt header module.
+//! # Autocrypt header module
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 
-use anyhow::{bail, Context as _, Error, Result};
 use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
 use crate::contact::addr_cmp;
+use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, SignedPublicKey};
 
@@ -37,13 +37,13 @@ impl fmt::Display for EncryptPreference {
 }
 
 impl str::FromStr for EncryptPreference {
-    type Err = Error;
+    type Err = ();
 
-    fn from_str(s: &str) -> Result<Self> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
             "nopreference" => Ok(EncryptPreference::NoPreference),
-            _ => bail!("Cannot parse encryption preference {}", s),
+            _ => Err(()),
         }
     }
 }
@@ -70,27 +70,28 @@ impl Aheader {
         }
     }
 
-    /// Tries to parse Autocrypt header.
-    ///
-    /// If there is none, returns None. If the header is present but cannot be parsed, returns an
-    /// error.
     pub fn from_headers(
+        context: &Context,
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
-    ) -> Result<Option<Self>> {
+    ) -> Option<Self> {
         if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
-            let header = Self::from_str(&value)?;
-            if !addr_cmp(&header.addr, wanted_from) {
-                bail!(
-                    "Autocrypt header address {:?} is not {:?}",
-                    header.addr,
-                    wanted_from
-                );
+            match Self::from_str(&value) {
+                Ok(header) => {
+                    if addr_cmp(&header.addr, wanted_from) {
+                        return Some(header);
+                    }
+                }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "found invalid autocrypt header {}: {:?}", value, err
+                    );
+                }
             }
-            Ok(Some(header))
-        } else {
-            Ok(None)
         }
+
+        None
     }
 }
 
@@ -119,9 +120,9 @@ impl fmt::Display for Aheader {
 }
 
 impl str::FromStr for Aheader {
-    type Err = Error;
+    type Err = ();
 
-    fn from_str(s: &str) -> Result<Self> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut attributes: BTreeMap<String, String> = s
             .split(';')
             .filter_map(|a| {
@@ -135,19 +136,15 @@ impl str::FromStr for Aheader {
 
         let addr = match attributes.remove("addr") {
             Some(addr) => addr,
-            None => bail!("Autocrypt header has no addr"),
+            None => {
+                return Err(());
+            }
         };
         let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .context("keydata attribute is not found")
-            .and_then(|raw| {
-                SignedPublicKey::from_base64(&raw).context("autocrypt key cannot be decoded")
-            })
-            .and_then(|key| {
-                key.verify()
-                    .and(Ok(key))
-                    .context("autocrypt key cannot be verified")
-            })?;
+            .ok_or(())
+            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
+            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -157,7 +154,7 @@ impl str::FromStr for Aheader {
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
         if attributes.keys().any(|k| !k.starts_with('_')) {
-            bail!("Unknown Autocrypt attribute found");
+            return Err(());
         }
 
         Ok(Aheader {
@@ -175,40 +172,38 @@ mod tests {
     const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
-    fn test_from_str() -> Result<()> {
+    fn test_from_str() {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
             RAWKEY
         )
-        .parse()?;
+        .parse()
+        .expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
-        Ok(())
     }
 
     // EncryptPreference::Reset is an internal value, parser should never return it
     #[test]
-    fn test_from_str_reset() -> Result<()> {
+    fn test_from_str_reset() {
         let raw = format!(
             "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
             RAWKEY
         );
-        let h: Aheader = raw.parse()?;
+        let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "reset@example.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
-        Ok(())
     }
 
     #[test]
-    fn test_from_str_non_critical() -> Result<()> {
+    fn test_from_str_non_critical() {
         let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
-        let h: Aheader = raw.parse()?;
+        let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
         assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
-        Ok(())
     }
 
     #[test]
@@ -221,7 +216,7 @@ mod tests {
     }
 
     #[test]
-    fn test_good_headers() -> Result<()> {
+    fn test_good_headers() {
         let fixed_header = concat!(
             "addr=a@b.example.org; prefer-encrypt=mutual; ",
             "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
@@ -247,7 +242,7 @@ mod tests {
             " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
         );
 
-        let ah = Aheader::from_str(fixed_header)?;
+        let ah = Aheader::from_str(fixed_header).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
         assert_eq!(format!("{}", ah), fixed_header);
@@ -255,17 +250,18 @@ mod tests {
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY))?;
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
             RAWKEY
-        ))?;
+        ))
+        .expect("failed to parse");
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))?;
-        Ok(())
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
+            .expect("failed to parse");
     }
 
     #[test]

src/blob.rs

@@ -1,28 +1,25 @@
-//! # Blob directory management.
+//! # Blob directory management
 
-use core::cmp::max;
 use std::ffi::OsStr;
 use std::fmt;
-use std::io::Cursor;
 
 use async_std::path::{Path, PathBuf};
 use async_std::prelude::*;
 use async_std::{fs, io};
 
-use anyhow::{format_err, Context as _, Error};
-use image::{DynamicImage, ImageFormat};
+use anyhow::Error;
+use image::GenericImageView;
 use num_traits::FromPrimitive;
 use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::{
-    MediaQuality, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE, WORSE_IMAGE_SIZE,
+    MediaQuality, Viewtype, BALANCED_AVATAR_SIZE, BALANCED_IMAGE_SIZE, WORSE_AVATAR_SIZE,
+    WORSE_IMAGE_SIZE,
 };
 use crate::context::Context;
 use crate::events::EventType;
-use crate::log::LogExt;
 use crate::message;
-use crate::message::Viewtype;
 
 /// Represents a file in the blob directory.
 ///
@@ -56,12 +53,12 @@ impl<'a> BlobObject<'a> {
     /// underlying error.
     pub async fn create(
         context: &'a Context,
-        suggested_name: &str,
+        suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name);
-        let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(blobdir, &stem, &ext).await?;
         file.write_all(data)
             .await
             .map_err(|err| BlobError::WriteFailure {
@@ -72,7 +69,7 @@ impl<'a> BlobObject<'a> {
 
         // workaround a bug in async-std
         // (the executor does not handle blocking operation in Drop correctly,
-        // see <https://github.com/async-rs/async-std/issues/900>)
+        // see https://github.com/async-rs/async-std/issues/900 )
         let _ = file.flush().await;
 
         let blob = BlobObject {
@@ -85,16 +82,13 @@ impl<'a> BlobObject<'a> {
 
     // Creates a new file, returning a tuple of the name and the handle.
     async fn create_new_file(
-        context: &Context,
         dir: &Path,
         stem: &str,
         ext: &str,
     ) -> Result<(String, fs::File), BlobError> {
-        const MAX_ATTEMPT: u32 = 16;
-        let mut attempt = 0;
+        let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
-        loop {
-            attempt += 1;
+        for attempt in 0..max_attempt {
             let path = dir.join(&name);
             match fs::OpenOptions::new()
                 .create_new(true)
@@ -104,20 +98,24 @@ impl<'a> BlobObject<'a> {
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
-                    if attempt >= MAX_ATTEMPT {
+                    if attempt == max_attempt {
                         return Err(BlobError::CreateFailure {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
                         });
-                    } else if attempt == 1 && !dir.exists().await {
-                        fs::create_dir_all(dir).await.ok_or_log(context);
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
                     }
                 }
             }
         }
+        // This is supposed to be unreachable, but the compiler doesn't know.
+        Err(BlobError::CreateFailure {
+            blobdir: dir.to_path_buf(),
+            blobname: name,
+            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+        })
     }
 
     /// Creates a new blob object with unique name by copying an existing file.
@@ -134,19 +132,20 @@ impl<'a> BlobObject<'a> {
     /// copied.
     pub async fn create_and_copy(
         context: &'a Context,
-        src: &Path,
+        src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file = fs::File::open(src)
-            .await
-            .map_err(|err| BlobError::CopyFailure {
-                blobdir: context.get_blobdir().to_path_buf(),
-                blobname: String::from(""),
-                src: src.to_path_buf(),
-                cause: err,
-            })?;
-        let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
+        let mut src_file =
+            fs::File::open(src.as_ref())
+                .await
+                .map_err(|err| BlobError::CopyFailure {
+                    blobdir: context.get_blobdir().to_path_buf(),
+                    blobname: String::from(""),
+                    src: src.as_ref().to_path_buf(),
+                    cause: err,
+                })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) =
-            BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
+            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
         let name_for_err = name.clone();
         if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
             {
@@ -157,7 +156,7 @@ impl<'a> BlobObject<'a> {
             return Err(BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
-                src: src.to_path_buf(),
+                src: src.as_ref().to_path_buf(),
                 cause: err,
             });
         }
@@ -191,13 +190,16 @@ impl<'a> BlobObject<'a> {
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
     pub async fn new_from_path(
-        context: &'a Context,
-        src: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        if src.starts_with(context.get_blobdir()) {
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'_>, BlobError> {
+        if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
-        } else if src.starts_with("$BLOBDIR/") {
-            BlobObject::from_name(context, src.to_str().unwrap_or_default().to_string())
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -218,22 +220,23 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WrongName] is used if the file name does not
     /// remain identical after sanitisation.
     pub fn from_path(
-        context: &'a Context,
-        path: &Path,
-    ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let rel_path =
-            path.strip_prefix(context.get_blobdir())
-                .map_err(|_| BlobError::WrongBlobdir {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    src: path.to_path_buf(),
-                })?;
-        if !BlobObject::is_acceptible_blob_name(rel_path) {
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::WrongBlobdir {
+                blobdir: context.get_blobdir().to_path_buf(),
+                src: path.as_ref().to_path_buf(),
+            })?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
-                blobname: path.to_path_buf(),
+                blobname: path.as_ref().to_path_buf(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
-            blobname: path.to_path_buf(),
+            blobname: path.as_ref().to_path_buf(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -289,7 +292,7 @@ impl<'a> BlobObject<'a> {
 
     /// Returns the filename of the blob.
     pub fn as_file_name(&self) -> &str {
-        self.name.rsplit('/').next().unwrap()
+        self.name.rsplitn(2, '/').next().unwrap()
     }
 
     /// The path relative in the blob directory.
@@ -302,7 +305,7 @@ impl<'a> BlobObject<'a> {
     /// If a blob's filename has an extension, it is always guaranteed
     /// to be lowercase.
     pub fn suffix(&self) -> Option<&str> {
-        let ext = self.name.rsplit('.').next();
+        let ext = self.name.rsplitn(2, '.').next();
         if ext == Some(&self.name) {
             None
         } else {
@@ -345,30 +348,13 @@ impl<'a> BlobObject<'a> {
         };
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
-        // Let's take the tricky filename
-        // "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
-        // Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
         let mut iter = clean.splitn(2, '.');
-
         let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        // stem == "file"
-
-        let ext_chars = iter.next().unwrap_or_default().chars();
-        let ext: String = ext_chars
-            .rev()
-            .take(32)
-            .collect::<Vec<_>>()
-            .iter()
-            .rev()
-            .collect();
-        // ext == "d_point_and_double_ending.tar.gz"
-
+        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
         if ext.is_empty() {
             (stem, "".to_string())
         } else {
             (stem, format!(".{}", ext).to_lowercase())
-            // Return ("file", ".d_point_and_double_ending.tar.gz")
-            // which is not perfect but acceptable.
         }
     }
 
@@ -394,7 +380,7 @@ impl<'a> BlobObject<'a> {
         true
     }
 
-    pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<(), BlobError> {
+    pub async fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
         let blob_abs = self.to_abs_path();
 
         let img_wh =
@@ -405,15 +391,7 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_AVATAR_SIZE,
             };
 
-        // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k.
-        // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k.
-        if let Some(new_name) = self
-            .recode_to_size(context, blob_abs, img_wh, Some(20_000))
-            .await?
-        {
-            self.name = new_name;
-        }
-        Ok(())
+        self.recode_to_size(context, blob_abs, img_wh).await
     }
 
     pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
@@ -432,70 +410,30 @@ impl<'a> BlobObject<'a> {
                 MediaQuality::Worse => WORSE_IMAGE_SIZE,
             };
 
-        if self
-            .recode_to_size(context, blob_abs, img_wh, None)
-            .await?
-            .is_some()
-        {
-            return Err(format_err!(
-                "Internal error: recode_to_size(..., None) shouldn't change the name of the image"
-            )
-            .into());
-        }
-        Ok(())
+        self.recode_to_size(context, blob_abs, img_wh).await
     }
 
     async fn recode_to_size(
         &self,
         context: &Context,
-        mut blob_abs: PathBuf,
-        mut img_wh: u32,
-        max_bytes: Option<usize>,
-    ) -> Result<Option<String>, BlobError> {
+        blob_abs: PathBuf,
+        img_wh: u32,
+    ) -> Result<(), BlobError> {
         let mut img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
         let orientation = self.get_exif_orientation(context);
-        let mut encoded = Vec::new();
-        let mut changed_name = None;
 
-        fn encode_img(img: &DynamicImage, encoded: &mut Vec<u8>) -> anyhow::Result<()> {
-            encoded.clear();
-            let mut buf = Cursor::new(encoded);
-            img.write_to(&mut buf, image::ImageFormat::Jpeg)?;
-            Ok(())
-        }
-        fn encoded_img_exceeds_bytes(
-            context: &Context,
-            img: &DynamicImage,
-            max_bytes: Option<usize>,
-            encoded: &mut Vec<u8>,
-        ) -> anyhow::Result<bool> {
-            if let Some(max_bytes) = max_bytes {
-                encode_img(img, encoded)?;
-                if encoded.len() > max_bytes {
-                    info!(
-                        context,
-                        "image size {}B ({}x{}px) exceeds {}B, need to scale down",
-                        encoded.len(),
-                        img.width(),
-                        img.height(),
-                        max_bytes,
-                    );
-                    return Ok(true);
-                }
-            }
-            Ok(false)
-        }
-        let exceeds_width = img.width() > img_wh || img.height() > img_wh;
-
-        let do_scale =
-            exceeds_width || encoded_img_exceeds_bytes(context, &img, max_bytes, &mut encoded)?;
+        let do_scale = img.width() > img_wh || img.height() > img_wh;
         let do_rotate = matches!(orientation, Ok(90) | Ok(180) | Ok(270));
 
         if do_scale || do_rotate {
+            if do_scale {
+                img = img.thumbnail(img_wh, img_wh);
+            }
+
             if do_rotate {
                 img = match orientation {
                     Ok(90) => img.rotate90(),
@@ -505,64 +443,14 @@ impl<'a> BlobObject<'a> {
                 }
             }
 
-            if do_scale {
-                if !exceeds_width {
-                    // The image is already smaller than img_wh, but exceeds max_bytes
-                    // We can directly start with trying to scale down to 2/3 of its current width
-                    img_wh = max(img.width(), img.height()) * 2 / 3
-                }
-
-                loop {
-                    let new_img = img.thumbnail(img_wh, img_wh);
-
-                    if encoded_img_exceeds_bytes(context, &new_img, max_bytes, &mut encoded)? {
-                        if img_wh < 20 {
-                            return Err(format_err!(
-                                "Failed to scale image to below {}B",
-                                max_bytes.unwrap_or_default()
-                            )
-                            .into());
-                        }
-
-                        img_wh = img_wh * 2 / 3;
-                    } else {
-                        if encoded.is_empty() {
-                            encode_img(&new_img, &mut encoded)?;
-                        }
-
-                        info!(
-                            context,
-                            "Final scaled-down image size: {}B ({}px)",
-                            encoded.len(),
-                            img_wh
-                        );
-                        break;
-                    }
-                }
-            }
-
-            // The file format is JPEG now, we may have to change the file extension
-            if !matches!(ImageFormat::from_path(&blob_abs), Ok(ImageFormat::Jpeg)) {
-                blob_abs = blob_abs.with_extension("jpg");
-                let file_name = blob_abs.file_name().context("No avatar file name (???)")?;
-                let file_name = file_name.to_str().context("Filename is no UTF-8 (???)")?;
-                changed_name = Some(format!("$BLOBDIR/{}", file_name));
-            }
-
-            if encoded.is_empty() {
-                encode_img(&img, &mut encoded)?;
-            }
-
-            fs::write(&blob_abs, &encoded)
-                .await
-                .map_err(|err| BlobError::WriteFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-                    cause: err.into(),
-                })?;
+            img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+                cause: err.into(),
+            })?;
         }
 
-        Ok(changed_name)
+        Ok(())
     }
 
     pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
@@ -626,23 +514,18 @@ pub enum BlobError {
     WrongBlobdir { blobdir: PathBuf, src: PathBuf },
     #[error("Blob has a badname {}", .blobname.display())]
     WrongName { blobname: PathBuf },
+    #[error("Sql: {0}")]
+    Sql(#[from] crate::sql::Error),
     #[error("{0}")]
     Other(#[from] anyhow::Error),
 }
 
 #[cfg(test)]
 mod tests {
-    use fs::File;
-
-    use anyhow::Result;
-    use image::{GenericImageView, Pixel};
-
-    use crate::chat::{self, create_group_chat, ProtectionStatus};
-    use crate::message::Message;
-    use crate::test_utils::{self, TestContext};
-
     use super::*;
 
+    use crate::test_utils::TestContext;
+
     #[async_std::test]
     async fn test_create() {
         let t = TestContext::new().await;
@@ -743,15 +626,13 @@ mod tests {
         let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
+        let blob = BlobObject::create_and_copy(&t, &src).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
-            .await
-            .is_err());
+        assert!(BlobObject::create_and_copy(&t, &whoops).await.is_err());
         let whoops = t.get_blobdir().join("whoops");
         assert!(!whoops.exists().await);
     }
@@ -762,9 +643,7 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).await.unwrap();
         assert_eq!(data, b"boo");
@@ -781,9 +660,7 @@ mod tests {
         let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
-            .await
-            .unwrap();
+        let blob = BlobObject::new_from_path(&t, &src_ext).await.unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -838,297 +715,4 @@ mod tests {
         assert!(!stem.contains('*'));
         assert!(!stem.contains('?'));
     }
-
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.jpg");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(&avatar_blob).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-
-        async fn file_size(path_buf: &PathBuf) -> u64 {
-            let file = File::open(path_buf).await.unwrap();
-            file.metadata().await.unwrap().len()
-        }
-
-        let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
-
-        blob.recode_to_size(&t, blob.to_abs_path(), 1000, Some(3000))
-            .await
-            .unwrap();
-        assert!(file_size(&avatar_blob).await <= 3000);
-        assert!(file_size(&avatar_blob).await > 2000);
-        let img = image::open(&avatar_blob).unwrap();
-        assert!(img.width() > 130);
-        assert_eq!(img.width(), img.height());
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.get_blobdir().join("avatar.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(test_utils::AVATAR_900x900_BYTES)
-            .await
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
-        assert_eq!(
-            avatar_cfg,
-            avatar_src.with_extension("jpg").to_str().unwrap()
-        );
-
-        let img = image::open(avatar_cfg).unwrap();
-        assert_eq!(img.width(), BALANCED_AVATAR_SIZE);
-        assert_eq!(img.height(), BALANCED_AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .await
-            .unwrap()
-            .write_all(avatar_bytes)
-            .await
-            .unwrap();
-        let avatar_blob = t.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[async_std::test]
-    async fn test_recode_image_1() {
-        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        // BALANCED_IMAGE_SIZE > 1000, the original image size, so the image is not scaled down:
-        send_image_check_mediaquality(Some("0"), bytes, 1000, 1000, 0, 1000, 1000)
-            .await
-            .unwrap();
-        send_image_check_mediaquality(
-            Some("1"),
-            bytes,
-            1000,
-            1000,
-            0,
-            WORSE_IMAGE_SIZE,
-            WORSE_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-    }
-
-    #[async_std::test]
-    async fn test_recode_image_2() {
-        // The "-rotated" files are rotated by 270 degrees using the Exif metadata
-        let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(
-            Some("0"),
-            bytes,
-            2000,
-            1800,
-            270,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let mut buf = Cursor::new(vec![]);
-        img_rotated
-            .write_to(&mut buf, image::ImageFormat::Jpeg)
-            .unwrap();
-        let bytes = buf.into_inner();
-
-        // Do this in parallel to speed up the test a bit
-        // (it still takes very long though)
-        let bytes2 = bytes.clone();
-        let join_handle = async_std::task::spawn(async move {
-            let img_rotated = send_image_check_mediaquality(
-                Some("0"),
-                &bytes2,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-                0,
-                BALANCED_IMAGE_SIZE * 1800 / 2000,
-                BALANCED_IMAGE_SIZE,
-            )
-            .await
-            .unwrap();
-            assert_correct_rotation(&img_rotated);
-        });
-
-        let img_rotated = send_image_check_mediaquality(
-            Some("1"),
-            &bytes,
-            BALANCED_IMAGE_SIZE * 1800 / 2000,
-            BALANCED_IMAGE_SIZE,
-            0,
-            WORSE_IMAGE_SIZE * 1800 / 2000,
-            WORSE_IMAGE_SIZE,
-        )
-        .await
-        .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        join_handle.await;
-    }
-
-    #[async_std::test]
-    async fn test_recode_image_3() {
-        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(Some("0"), bytes, 200, 180, 270, 180, 200)
-            .await
-            .unwrap();
-        assert_correct_rotation(&img_rotated);
-
-        let bytes = include_bytes!("../test-data/image/rectangle200x180-rotated.jpg");
-        let img_rotated = send_image_check_mediaquality(Some("1"), bytes, 200, 180, 270, 180, 200)
-            .await
-            .unwrap();
-        assert_correct_rotation(&img_rotated);
-    }
-
-    fn assert_correct_rotation(img: &DynamicImage) {
-        // The test images are black in the bottom left corner after correctly applying
-        // the EXIF orientation
-
-        let [luma] = img.get_pixel(10, 10).to_luma().0;
-        assert_eq!(luma, 255);
-        let [luma] = img.get_pixel(img.width() - 10, 10).to_luma().0;
-        assert_eq!(luma, 255);
-        let [luma] = img
-            .get_pixel(img.width() - 10, img.height() - 10)
-            .to_luma()
-            .0;
-        assert_eq!(luma, 255);
-        let [luma] = img.get_pixel(10, img.height() - 10).to_luma().0;
-        assert_eq!(luma, 0);
-    }
-
-    async fn send_image_check_mediaquality(
-        media_quality_config: Option<&str>,
-        bytes: &[u8],
-        original_width: u32,
-        original_height: u32,
-        orientation: i32,
-        compressed_width: u32,
-        compressed_height: u32,
-    ) -> anyhow::Result<DynamicImage> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        alice
-            .set_config(Config::MediaQuality, media_quality_config)
-            .await?;
-        let file = alice.get_blobdir().join("file.jpg");
-
-        File::create(&file).await?.write_all(bytes).await?;
-        let img = image::open(&file)?;
-        assert_eq!(img.width(), original_width);
-        assert_eq!(img.height(), original_height);
-
-        let blob = BlobObject::new_from_path(&alice, &file).await?;
-        assert_eq!(blob.get_exif_orientation(&alice).unwrap_or(0), orientation);
-
-        let mut msg = Message::new(Viewtype::Image);
-        msg.set_file(file.to_str().unwrap(), None);
-        let chat = alice.create_chat(&bob).await;
-        let sent = alice.send_msg(chat.id, &mut msg).await;
-        let alice_msg = alice.get_last_msg().await;
-        assert_eq!(alice_msg.get_width() as u32, compressed_width);
-        assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let img = image::open(alice_msg.get_file(&alice).unwrap())?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
-
-        bob.recv_msg(&sent).await;
-        let bob_msg = bob.get_last_msg().await;
-        assert_eq!(bob_msg.get_width() as u32, compressed_width);
-        assert_eq!(bob_msg.get_height() as u32, compressed_height);
-        let file = bob_msg.get_file(&bob).unwrap();
-
-        let blob = BlobObject::new_from_path(&bob, &file).await?;
-        assert_eq!(blob.get_exif_orientation(&bob).unwrap_or(0), 0);
-
-        let img = image::open(file)?;
-        assert_eq!(img.width() as u32, compressed_width);
-        assert_eq!(img.height() as u32, compressed_height);
-        Ok(img)
-    }
-
-    #[async_std::test]
-    async fn test_increation_in_blobdir() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
-
-        let file = t.get_blobdir().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file(file.to_str().unwrap(), None);
-        let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
-        assert_eq!(prepared_id, msg.id);
-        assert!(msg.is_increation());
-
-        let msg = Message::load_from_db(&t, prepared_id).await?;
-        assert!(msg.is_increation());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_increation_not_blobdir() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
-        assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
-
-        let file = t.dir.path().join("anyfile.dat");
-        File::create(&file).await?.write_all("bla".as_ref()).await?;
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file(file.to_str().unwrap(), None);
-        assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
-
-        Ok(())
-    }
 }