python: fail fast on the tests

Do not waste CI time running the rest of the tests
if CI is not going to be green anyway.

.github/mergeable.yml

@@ -5,22 +5,22 @@ mergeable:
     validate:
       - do: or
         validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/src/**'
-                required: ['CHANGELOG.md']
+          - do: description
+            must_include:
+              regex: "#skip-changelog"
+          - do: and
+            validate:
+              - do: dependent
+                changed:
+                  file: "src/**"
+                  required: ["CHANGELOG.md"]
+              - do: dependent
+                changed:
+                  file: "deltachat-ffi/src/**"
+                  required: ["CHANGELOG.md"]
     fail:
       - do: checks
-        status: 'action_required' 
+        status: "action_required"
         payload:
           title: Changelog might need an update
           summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -1,48 +1,55 @@
 name: Rust CI
 
+# Cancel previously started workflow runs
+# when the branch is updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
   push:
     branches:
       - master
-      - staging
-      - trying
 
 env:
   RUSTFLAGS: -Dwarnings
-   
+
 jobs:
+  lint:
+    name: Rustfmt and Clippy
+    runs-on: ubuntu-latest
+    env:
+      RUSTUP_TOOLCHAIN: 1.68.0
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install rustfmt and clippy
+        run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v2
+      - name: Run rustfmt
+        run: cargo fmt --all -- --check
+      - name: Run clippy
+        run: scripts/clippy.sh
 
-  fmt:
-    name: Rustfmt
+  cargo_deny:
+    name: cargo deny
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: EmbarkStudios/cargo-deny-action@v1
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v2
-      - run: cargo fmt --all -- --check
+          arguments: --all-features --workspace
+          command: check
+          command-arguments: "-Dwarnings"
 
-  run_clippy:
+  provider_database:
+    name: Check provider database
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: stable
-            components: clippy
-            override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v2
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches --features repl -- -D warnings
+      - name: Check provider database
+        run: scripts/update-provider-database.sh
 
   docs:
     name: Rust doc comments
@@ -52,13 +59,6 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
       - name: Rustdoc
@@ -67,95 +67,94 @@ jobs:
   build_and_test:
     name: Build and test
     strategy:
+      fail-fast: false
       matrix:
         include:
           # Currently used Rust version.
           - os: ubuntu-latest
-            rust: 1.64.0
+            rust: 1.68.0
             python: 3.9
           - os: windows-latest
-            rust: 1.64.0
+            rust: 1.68.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.63.0
+          # Minimum Supported Rust Version = 1.64.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.63.0
+            rust: 1.64.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@master
+      - uses: actions/checkout@master
 
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
+      - name: Install Rust ${{ matrix.rust }}
+        run: rustup toolchain install --profile minimal ${{ matrix.rust }}
+      - run: rustup override set ${{ matrix.rust }}
 
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v2
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v2
 
-    - name: check
-      run: cargo check --all --bins --examples --tests --features repl --benches
+      - name: Check
+        run: cargo check --workspace --bins --examples --tests --benches
 
-    - name: tests
-      run: cargo test --all
+      - name: Tests
+        run: cargo test --workspace
 
-    - name: test cargo vendor
-      run: cargo vendor
+      - name: Test cargo vendor
+        run: cargo vendor
 
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Install python
+        if: ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
+      - name: Install tox
+        if: ${{ matrix.python }}
+        run: pip install tox
 
-    - name: build C library
-      if: ${{ matrix.python }}
-      run: cargo build -p deltachat_ffi --features jsonrpc
+      - name: Build C library
+        if: ${{ matrix.python }}
+        run: cargo build -p deltachat_ffi --features jsonrpc
 
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3
+      - name: Run python tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          DCC_RS_TARGET: debug
+          DCC_RS_DEV: ${{ github.workspace }}
+        working-directory: python
+        run: tox -e lint,mypy,doc,py3
 
-    - name: build deltachat-rpc-server
-      if: ${{ matrix.python }}
-      run: cargo build -p deltachat-rpc-server
+      - name: Build deltachat-rpc-server
+        if: ${{ matrix.python }}
+        run: cargo build -p deltachat-rpc-server
 
-    - name: add deltachat-rpc-server to path
-      if: ${{ matrix.python }}
-      run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
+      - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.python }}
+        run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
 
-    - name: run deltachat-rpc-client tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-      working-directory: deltachat-rpc-client
-      run: tox -e py3
+      - name: Run deltachat-rpc-client tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        working-directory: deltachat-rpc-client
+        run: tox -e py3,lint
 
-    - name: install pypy
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: 'pypy${{ matrix.python }}'
+      - name: Install pypy
+        if: ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "pypy${{ matrix.python }}"
 
-    - name: run pypy tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e pypy3
+      - name: Run pypy tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          DCC_RS_TARGET: debug
+          DCC_RS_DEV: ${{ github.workspace }}
+        working-directory: python
+        run: tox -e pypy3

.github/workflows/deltachat-rpc-server.yml

@@ -0,0 +1,126 @@
+# Manually triggered action to build deltachat-rpc-server binaries.
+
+name: Build deltachat-rpc-server binaries
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  # Build a version statically linked against musl libc
+  # to avoid problems with glibc version incompatibility.
+  build_linux:
+    name: Cross-compile deltachat-rpc-server for x86_64, aarch64 and armv7 Linux
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build
+        run: sh scripts/zig-rpc-server.sh
+
+      - name: Upload x86_64 binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-x86_64
+          path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload aarch64 binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-aarch64
+          path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload armv7 binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-armv7
+          path: target/armv7-unknown-linux-musleabihf/release/deltachat-rpc-server
+          if-no-files-found: error
+
+  build_windows:
+    name: Build deltachat-rpc-server for Windows
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            artifact: win32.exe
+            path: deltachat-rpc-server.exe
+            target: i686-pc-windows-msvc
+
+          - os: windows-latest
+            artifact: win64.exe
+            path: deltachat-rpc-server.exe
+            target: x86_64-pc-windows-msvc
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup rust target
+        run: rustup target add ${{ matrix.target }}
+
+      - name: Build
+        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-${{ matrix.artifact }}
+          path: target/${{ matrix.target}}/release/${{ matrix.path }}
+          if-no-files-found: error
+
+  publish:
+    name: Upload binaries to the release
+    needs: ["build_linux", "build_windows"]
+    permissions:
+      contents: write
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: Download deltachat-rpc-server-x86_64
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "deltachat-rpc-server-x86_64"
+          path: "dist/deltachat-rpc-server-x86_64"
+
+      - name: Download deltachat-rpc-server-aarch64
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "deltachat-rpc-server-aarch64"
+          path: "dist/deltachat-rpc-server-aarch64"
+
+      - name: Download deltachat-rpc-server-armv7
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "deltachat-rpc-server-armv7"
+          path: "dist/deltachat-rpc-server-armv7"
+
+      - name: Download deltachat-rpc-server-win32.exe
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "deltachat-rpc-server-win32.exe"
+          path: "dist/deltachat-rpc-server-win32.exe"
+
+      - name: Download deltachat-rpc-server-win64.exe
+        uses: "actions/download-artifact@v3"
+        with:
+          name: "deltachat-rpc-server-win64.exe"
+          path: "dist/deltachat-rpc-server-win64.exe"
+
+      - name: List downloaded artifacts
+        run: ls -l dist/
+
+      - name: Upload binaries to the GitHub release
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh release upload ${{ github.ref_name }} \
+            --repo ${{ github.repository }} \
+            dist/deltachat-rpc-server-*

.github/workflows/jsonrpc-client-npm-package.yml

@@ -1,29 +1,28 @@
-name: 'jsonrpc js client build'
+name: "jsonrpc js client build"
 on:
   pull_request:
   push:
     tags:
-    - '*'
-    - '!py-*'
-
+      - "*"
+      - "!py-*"
 
 jobs:
   pack-module:
-    name: 'Package @deltachat/jsonrpc-client and upload to download.delta.chat'
-    runs-on: ubuntu-18.04
+    name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat"
+    runs-on: ubuntu-20.04
     steps:
-      - name: install tree
+      - name: Install tree
         run: sudo apt install tree
       - name: Checkout
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '16'
-      - name: get tag
+          node-version: "16"
+      - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
         continue-on-error: true
-      - name: Get Pullrequest ID
+      - name: Get Pull Request ID
         id: prepare
         run: |
           tag=${{ steps.tag.outputs.tag }}
@@ -38,11 +37,11 @@ jobs:
           npm --version
           node --version
           echo $DELTACHAT_JSONRPC_TAR_GZ
-      - name: install dependencies without running scripts
+      - name: Install dependencies without running scripts
         run: |
           cd deltachat-jsonrpc/typescript
           npm install --ignore-scripts
-      - name: package
+      - name: Package
         shell: bash
         run: |
           cd deltachat-jsonrpc/typescript
@@ -65,13 +64,13 @@ jobs:
           chmod 600 __TEMP_INPUT_KEY_FILE
           scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
         continue-on-error: true
-      - name: "Post links to details"
+      - name: Post links to details
         if: steps.upload-preview.outcome == 'success'
         run: node ./node/scripts/postLinksToDetails.js
         env:
-         URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-         MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
+          URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
       # Upload to download.delta.chat/node/
       - name: Upload deltachat-jsonrpc-client build to download.delta.chat/node/
         if: ${{ steps.tag.outputs.tag }}

.github/workflows/jsonrpc.yml

@@ -35,6 +35,10 @@ jobs:
           npm run test
         env:
           DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+      - name: make sure websocket server version still builds
+        run: |
+          cd deltachat-jsonrpc
+          cargo build --bin deltachat-jsonrpc-server --features webserver
       - name: Run linter
         run: |
           cd deltachat-jsonrpc/typescript

.github/workflows/node-delete-preview.yml

@@ -7,26 +7,25 @@ on:
 
 jobs:
   delete:
-
     runs-on: ubuntu-latest
 
     steps:
-    - name: Get Pullrequest ID
-      id: getid
-      run: |
-        export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
-        echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
-    - name: Renaming
-      run: |
-        # create empty file to copy it over the outdated deliverable on download.delta.chat
-        echo "This preview build is outdated and has been removed." > empty
-        cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
-    - name: Replace builds with dummy files
-      uses: horochx/deploy-via-scp@v1.0.1
-      with:
-        user: ${{ secrets.USERNAME }}
-        key: ${{ secrets.SSH_KEY }}
-        host: "download.delta.chat"
-        port: 22
-        local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
-        remote: "/var/www/html/download/node/preview/"
+      - name: Get Pull Request ID
+        id: getid
+        run: |
+          export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
+          echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
+      - name: Renaming
+        run: |
+          # create empty file to copy it over the outdated deliverable on download.delta.chat
+          echo "This preview build is outdated and has been removed." > empty
+          cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
+      - name: Replace builds with dummy files
+        uses: horochx/deploy-via-scp@v1.0.1
+        with:
+          user: ${{ secrets.USERNAME }}
+          key: ${{ secrets.SSH_KEY }}
+          host: "download.delta.chat"
+          port: 22
+          local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
+          remote: "/var/www/html/download/node/preview/"

.github/workflows/node-docs.yml

@@ -9,26 +9,26 @@ jobs:
   generate:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v3
 
-    - name: Use Node.js 16.x
-      uses: actions/setup-node@v3
-      with:
-        node-version: 16.x
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
 
-    - name: npm install and generate documentation
-      run: |
-        cd node
-        npm i --ignore-scripts
-        npx typedoc
-        mv docs js
+      - name: npm install and generate documentation
+        run: |
+          cd node
+          npm i --ignore-scripts
+          npx typedoc
+          mv docs js
 
-    - name: Upload
-      uses: horochx/deploy-via-scp@v1.0.1
-      with:
-        user: ${{ secrets.USERNAME }}
-        key: ${{ secrets.KEY }}
-        host: "delta.chat"
-        port: 22
-        local: "node/js"
-        remote: "/var/www/html/"
+      - name: Upload
+        uses: horochx/deploy-via-scp@v1.0.1
+        with:
+          user: ${{ secrets.USERNAME }}
+          key: ${{ secrets.KEY }}
+          host: "delta.chat"
+          port: 22
+          local: "node/js"
+          remote: "/var/www/html/"

.github/workflows/node-package.yml

@@ -1,25 +1,24 @@
-name: 'node.js build'
+name: "node.js build"
 on:
   pull_request:
   push:
     tags:
-    - '*'
-    - '!py-*'
-
+      - "*"
+      - "!py-*"
 
 jobs:
   prebuild:
-    name: 'prebuild'
+    name: Prebuild
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos-latest, windows-latest]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: "16"
       - name: System info
         run: |
           rustc -vV
@@ -65,21 +64,21 @@ jobs:
 
   pack-module:
     needs: prebuild
-    name: 'Package deltachat-node and upload to download.delta.chat'
-    runs-on: ubuntu-18.04
+    name: Package deltachat-node and upload to download.delta.chat
+    runs-on: ubuntu-20.04
     steps:
-      - name: install tree
+      - name: Install tree
         run: sudo apt install tree
       - name: Checkout
         uses: actions/checkout@v3
       - uses: actions/setup-node@v2
         with:
-          node-version: '16'
-      - name: get tag
+          node-version: "16"
+      - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
         continue-on-error: true
-      - name: Get Pullrequest ID
+      - name: Get Pull Request ID
         id: prepare
         run: |
           tag=${{ steps.tag.outputs.tag }}
@@ -97,43 +96,43 @@ jobs:
           npm --version
           node --version
           echo $DELTACHAT_NODE_TAR_GZ
-      - name: Download ubuntu prebuild
+      - name: Download Ubuntu prebuild
         uses: actions/download-artifact@v1
         with:
-          name: ubuntu-18.04
-      - name: Download macos prebuild
+          name: ubuntu-20.04
+      - name: Download macOS prebuild
         uses: actions/download-artifact@v1
         with:
           name: macos-latest
-      - name: Download windows prebuild
+      - name: Download Windows prebuild
         uses: actions/download-artifact@v1
         with:
           name: windows-latest
       - shell: bash
         run: |
           mkdir node/prebuilds
-          tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C node/prebuilds
+          tar -xvzf ubuntu-20.04/ubuntu-20.04.tar.gz -C node/prebuilds
           tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
           tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
           tree node/prebuilds
-          rm -rf ubuntu-18.04 macos-latest windows-latest
-      - name: install dependencies without running scripts
+          rm -rf ubuntu-20.04 macos-latest windows-latest
+      - name: Install dependencies without running scripts
         run: |
           npm install --ignore-scripts
-      - name: build constants
+      - name: Build constants
         run: |
           npm run build:core:constants
-      - name: build typescript part
+      - name: Build TypeScript part
         run: |
           npm run build:bindings:ts
-      - name: package
+      - name: Package
         shell: bash
         run: |
           mv node/README.md README.md
           npm pack .
           ls -lah
           mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
-      - name: Upload Prebuild
+      - name: Upload prebuild
         uses: actions/upload-artifact@v3
         with:
           name: deltachat-node.tgz
@@ -148,12 +147,12 @@ jobs:
           chmod 600 __TEMP_INPUT_KEY_FILE
           scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
         continue-on-error: true
-      - name: "Post links to details"
+      - name: Post links to details
         if: steps.upload-preview.outcome == 'success'
         run: node ./node/scripts/postLinksToDetails.js
         env:
-         URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Upload to download.delta.chat/node/
       - name: Upload deltachat-node build to download.delta.chat/node/
         if: ${{ steps.tag.outputs.tag }}

.github/workflows/node-tests.yml

@@ -1,25 +1,30 @@
-name: 'node.js tests'
+name: "node.js tests"
+
+# Cancel previously started workflow runs
+# when the branch is updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
   push:
     branches:
       - master
-      - staging
-      - trying
 
 jobs:
   tests:
-    name: 'tests'
+    name: Tests
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '16'
+          node-version: "16"
       - name: System info
         run: |
           rustc -vV
@@ -59,7 +64,7 @@ jobs:
           npm run test
         env:
           DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-          NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
+          NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"
       - name: Run tests on Windows, except lint
         timeout-minutes: 10
         if: runner.os == 'Windows'
@@ -68,4 +73,4 @@ jobs:
           npm run test:mocha
         env:
           DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-          NODE_OPTIONS: '--force-node-api-uncaught-exceptions-policy=true'
+          NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/repl.yml

@@ -11,19 +11,13 @@ jobs:
     name: Build REPL example
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+      - uses: actions/checkout@v3
 
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.66.0
-        override: true
+      - name: Build
+        run: cargo build -p deltachat-repl --features vendored
 
-    - name: build
-      run: cargo build --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v3
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: repl.exe
+          path: "target/debug/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -8,20 +8,18 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat --no-deps
-    - name: Upload to rs.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/rs/"
-
+      - uses: actions/checkout@v3
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat --no-deps --document-private-items
+      - name: Upload to rs.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: ${{ secrets.USERNAME }}
+          KEY: ${{ secrets.KEY }}
+          HOST: "delta.chat"
+          SOURCE: "target/doc"
+          TARGET: "/var/www/html/rs/"

.github/workflows/upload-ffi-docs.yml

@@ -8,20 +8,18 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Build the documentation with cargo
-      run: |
-        cargo doc --package deltachat_ffi --no-deps
-    - name: Upload to cffi.delta.chat
-      uses: up9cloud/action-rsync@v1.3
-      env:
-        USER: ${{ secrets.USERNAME }}
-        KEY: ${{ secrets.KEY }}
-        HOST: "delta.chat"
-        SOURCE: "target/doc"
-        TARGET: "/var/www/html/cffi/"
-
+      - uses: actions/checkout@v3
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat_ffi --no-deps
+      - name: Upload to cffi.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: ${{ secrets.USERNAME }}
+          KEY: ${{ secrets.KEY }}
+          HOST: "delta.chat"
+          SOURCE: "target/doc"
+          TARGET: "/var/www/html/cffi/"

CHANGELOG.md

@@ -1,12 +1,157 @@
 # Changelog
 
-## Unreleased
+## [Unreleased]
 
 ### Changes
+- "full message view" not needed because of footers that go to contact status #4151
+- Pick up system's light/dark mode in generated message HTML #4150
+- Support non-persistent configuration with DELTACHAT_* env
+- Print deltachat-repl errors with causes. #4166
+- Increase MSRV to 1.64. #4167
+- Core takes care of stopping and re-starting IO itself where needed,
+  e.g. during backup creation.  It is no longer needed to call
+  dc_stop_io().  dc_start_io() can now be called at any time without
+  harm. #4138
+- More accurate maybe_add_bcc_self device message text #4175
+
+### Fixes
+- Fix segmentation fault if `dc_context_unref()` is called during
+  background process spawned by `dc_configure()` or `dc_imex()`
+  or `dc_jsonrpc_instance_t` is unreferenced
+  during handling the JSON-RPC request. #4153
+- Delete expired messages using multiple SQL requests. #4158
+- Do not emit "Failed to run incremental vacuum" warnings on success. #4160
+- Ability to send backup over network and QR code to setup second device #4007
+- Disable buffering during STARTTLS setup. #4190
+
+## [1.111.0] - 2023-03-05
+
+### Changes
+- Make smeared timestamp generation non-async. #4075
+- Set minimum TLS version to 1.2. #4096
+- Run `cargo-deny` in CI. #4101
+- Check provider database with CI. #4099 
+- Switch to DEFERRED transactions #4100
+
+### Fixes
+- Do not block async task executor while decrypting the messages. #4079
+- Housekeeping: delete the blobs backup dir #4123
+
+### API-Changes
+- jsonrpc: add more advanced API to send a message. #4097
+- jsonrpc: add get webxdc blob API `getWebxdcBlob` #4070
+
+
+## 1.110.0
+
+### Changes
+- use transaction in `Contact::add_or_lookup()` #4059
+- Organize the connection pool as a stack rather than a queue to ensure that
+  connection page cache is reused more often.
+  This speeds up tests by 28%, real usage will have lower speedup. #4065
+- Use transaction in `update_blocked_mailinglist_contacts`. #4058
+- Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055
+- Updated provider database.
+- Disable DKIM-Checks again #4076
+- Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089
+- mimeparser: handle headers from the signed part of unencrypted signed message #4013
+
+### Fixes
+- Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063
+- Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted.
+  Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972
+- Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067
+- Update timestamps in `param` columns with transactions. #4083
 
 ### API-Changes
 
+
+## 1.109.0
+
+### Changes
+- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
+
 ### Fixes
+- deltachat-rpc-server: do not block stdin while processing the request. #4041
+  deltachat-rpc-server now reads the next request as soon as previous request handler is spawned.
+- Enable `auto_vacuum` on all SQL connections. #2955
+- Replace `r2d2` connection pool with an own implementation. #4050 #4053 #4043 #4061
+  This change improves reliability
+  by closing all database connections immediately when the context is closed.
+
+### API-Changes
+
+- Remove `MimeMessage::from_bytes()` public interface. #4033
+- BREAKING Types: jsonrpc: `get_messages` now returns a map with `MessageLoadResult` instead of failing completely if one of the requested messages could not be loaded. #4038
+- Add `dc_msg_set_subject()`. C-FFI #4057
+- Mark python bindings as supporting typing according to PEP 561 #4045
+
+
+## 1.108.0
+
+### Changes
+- Use read/write timeouts instead of per-command timeouts for SMTP #3985
+- Cache DNS results for SMTP connections #3985
+- Prefer TLS over STARTTLS during autoconfiguration #4021
+- Use SOCKS5 configuration for HTTP requests #4017
+- Show non-deltachat emails by default for new installations #4019
+- Re-enabled SMTP pipelining after disabling it in #4006
+
+### Fixes
+- Fix Securejoin for multiple devices on a joining side #3982
+- python: handle NULL value returned from `dc_get_msg()` #4020
+  Account.`get_message_by_id` may return `None` in this case.
+
+### API-Changes
+- Remove bitflags from `get_chat_msgs()` interface #4022
+  C interface is not changed.
+  Rust and JSON-RPC API have `flags` integer argument
+  replaced with two boolean flags `info_only` and `add_daymarker`.
+- jsonrpc: add API to check if the message is sent by a bot #3877
+
+
+## 1.107.1
+
+### Changes
+- Log server security (TLS/STARTTLS/plain) type #4005
+
+### Fixes
+- Disable SMTP pipelining #4006
+
+
+## 1.107.0
+
+### Changes
+- Pipeline SMTP commands #3924
+- Cache DNS results for IMAP connections #3970
+
+### Fixes
+- Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914
+- fix verifier-by addr was empty string instead of None #3961
+- Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with
+  unread messages increases #3959
+- Fix Peerstate comparison #3962
+- Log SOCKS5 configuration for IMAP like already done for SMTP #3964
+- Fix SOCKS5 usage for IMAP #3965
+- Exit from recently seen loop on interrupt channel errors to avoid busy looping #3966
+
+### API-Changes
+- jsonrpc: add verified-by information to `Contact`-Object
+- Remove `attach_selfavatar` config #3951
+
+### Changes
+- add debug logging support for webxdcs #3296
+
+## 1.106.0
+
+### Changes
+- Only send IncomingMsgBunch if there are more than 0 new messages #3941
+
+### Fixes
+- fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938
+- Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943
+- Do not treat invalid email addresses as an exception #3942
+- Add timeouts to HTTP requests #3948
 
 
 ## 1.105.0
@@ -83,7 +228,7 @@
 - jsonrpc: Add async Python client #3734
 
 ### Fixes
-- Make sure malformed messsages will never block receiving further messages anymore #3771
+- Make sure malformed messages will never block receiving further messages anymore #3771
 - strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650
 - Assume all Thunderbird users prefer encryption #3774
 - refactor peerstate handling to ensure no duplicate peerstates #3776
@@ -235,7 +380,7 @@
   - `importBackup()`
   - `getMessageHtml()` #3671
   - `miscGetStickerFolder` and `miscGetStickers` #3672
-- breaking: jsonrpc: remove function `messageListGetMessageIds()`, it is replaced by `getMessageIds()` and `getMessageListItems()` the latter returns a new `MessageListItem` type, which is the now prefered way of using the message list.
+- breaking: jsonrpc: remove function `messageListGetMessageIds()`, it is replaced by `getMessageIds()` and `getMessageListItems()` the latter returns a new `MessageListItem` type, which is the now preferred way of using the message list.
 - jsonrpc: add type: #3641, #3645
   - `MessageSearchResult`
   - `Location`
@@ -261,7 +406,7 @@
 - jsonrpc js client:
   - Change package name from `deltachat-jsonrpc-client` to `@deltachat/jsonrpc-client`
   - remove relative file dependency to it from `deltachat-node` (because it did not work anyway and broke the nix build of desktop)
-  - ci: add github ci action to upload it to our download server automaticaly on realease
+  - ci: add github ci action to upload it to our download server automatically on release
 
 ## 1.95.0
 
@@ -338,7 +483,7 @@
 - Auto accept contact requests if `Config::Bot` is set for a client #3567 
 - Don't prepend the subject to chat messages in mailinglists
 - fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585
-- Reject webxcd-updates from contacts who are not group members #3568
+- Reject webxdc-updates from contacts who are not group members #3568
 
 
 ## 1.93.0
@@ -561,7 +706,7 @@
 
 ### Fixes
 - node: throw error when getting context with an invalid account id
-- node: throw error when instanciating a wrapper class on `null` (Context, Message, Chat, ChatList and so on)
+- node: throw error when instantiating a wrapper class on `null` (Context, Message, Chat, ChatList and so on)
 - use same contact-color if email address differ only in upper-/lowercase #3327
 - repair encrypted mails "mixed up" by Google Workspace "Append footer" function #3315
 
@@ -649,7 +794,7 @@
 - hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
 - do not delete messages without Message-IDs as duplicates #3095
 - assign replies from a different email address to the correct chat #3119
-- assing outgoing private replies to the correct chat #3177
+- assign outgoing private replies to the correct chat #3177
 - start ephemeral timer when seen status is synchronized via IMAP #3122
 - do not create empty contact requests with "setup changed" messages;
   instead, send a "setup changed" message into all chats we share with the peer #3187
@@ -702,7 +847,7 @@
 - don't watch Sent folder by default #3025
 - use webxdc app name in chatlist/quotes/replies etc. #3027
 - make it possible to cancel message sending by removing the message #3034,
-  this was previosuly removed in 1.71.0 #2939
+  this was previously removed in 1.71.0 #2939
 - synchronize Seen flags only on watched folders to speed up
   folder scanning #3041
 - remove direct dependency on `byteorder` crate #3031
@@ -1924,7 +2069,7 @@
 - #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
 
 - #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
-  improvments 
+  improvements
 
 ## 1.0.0-beta.16
 
@@ -2003,7 +2148,7 @@
 - trigger reconnect more often on imap error states.  Should fix an 
   issue observed when trying to empty a folder.  @hpk42
 
-- un-split qr tests: we fixed qr-securejoin protocol flakyness 
+- un-split qr tests: we fixed qr-securejoin protocol flakiness 
   last weeks. @hpk42
 
 ## 1.0.0-beta.10
@@ -2041,7 +2186,7 @@
 
 - fix moving self-sent messages, thanks @r10s, @flub, @hpk42
 
-- fix flakyness/sometimes-failing verified/join-protocols, 
+- fix flakiness/sometimes-failing verified/join-protocols, 
   thanks @flub, @r10s, @hpk42
 
 - fix reply-to-encrypted message to keep encryption 
@@ -2060,7 +2205,7 @@
 
 - fixes imap-protocol parsing bugs that lead to infinitely
   repeated crashing while trying to receive messages with
-  a subjec that contained non-utf8. thanks @link2xt
+  a subject that contained non-utf8. thanks @link2xt
 
 - fixed logic to find encryption subkey -- previously 
   delta chat would use the primary key for encryption
@@ -2168,3 +2313,6 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+
+[unreleased]: https://github.com/deltachat/deltachat-core-rust/compare/v1.111.0...HEAD
+[1.111.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.110.0...v1.111.0

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.105.0"
+version = "1.111.0"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.63"
+rust-version = "1.64"
 
 [profile.dev]
 debug = 0
@@ -13,54 +13,67 @@ opt-level = 1
 [profile.test]
 opt-level = 0
 
+# Always optimize dependencies.
+# This does not apply to crates in the workspace.
+# <https://doc.rust-lang.org/cargo/reference/profiles.html#overrides>
+[profile.dev.package."*"]
+opt-level = "z"
+
 [profile.release]
 lto = true
 panic = 'abort'
+opt-level = "z"
+
+[patch.crates-io]
+default-net = { git = "https://github.com/dignifiedquire/default-net.git", branch="feat-android" }
+quinn-udp = { git = "https://github.com/quinn-rs/quinn", branch="main" }
+quinn-proto = { git = "https://github.com/quinn-rs/quinn", branch="main" }
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 format-flowed = { path = "./format-flowed" }
+ratelimit = { path = "./deltachat-ratelimit" }
 
-ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
+async-channel = "1.8.0"
 async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
-async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
-async-smtp = { version = "0.5", default-features = false, features = ["smtp-transport", "socks5", "runtime-tokio"] }
-trust-dns-resolver = "0.22"
-tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
-tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
+async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
 backtrace = "0.3"
-base64 = "0.20"
+base64 = "0.21"
 bitflags = "1.3"
 chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
-dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
+fast-socks5 = "0.8"
 futures = "0.3"
+futures-lite = "1.12.0"
 hex = "0.4.0"
+humansize = "2"
 image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+# iroh = { version = "0.3.0", default-features = false }
+iroh = { git = 'https://github.com/n0-computer/iroh', branch = "flub/ticket-multiple-addrs" }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-log = {version = "0.4.16", optional = true }
 mailparse = "0.14"
-native-tls = "0.2"
 num_cpus = "1.15"
 num-derive = "0.3"
 num-traits = "0.2"
 once_cell = "1.17.0"
 percent-encoding = "2.2"
+parking_lot = "0.12"
 pgp = { version = "0.9", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
+qrcodegen = "1.7.0"
 quick-xml = "0.27"
-r2d2 = "0.8"
-r2d2_sqlite = "0.20"
 rand = "0.8"
 regex = "1.7"
-rusqlite = { version = "0.27", features = ["sqlcipher"] }
+reqwest = { version = "0.11.14", features = ["json"] }
+rusqlite = { version = "0.28", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustyline = { version = "10", optional = true }
 sanitize-filename = "0.4"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -69,21 +82,17 @@ sha2 = "0.10"
 smallvec = "1"
 strum = "0.24"
 strum_macros = "0.24"
-thiserror = "1"
-toml = "0.5"
-url = "2"
-uuid = { version = "1", features = ["serde", "v4"] }
-fast-socks5 = "0.8"
-humansize = "2"
-qrcodegen = "1.7.0"
 tagger = "4.3.4"
 textwrap = "0.16.0"
-async-channel = "1.8.0"
-futures-lite = "1.12.0"
-tokio-stream = { version = "0.1.11", features = ["fs"] }
+thiserror = "1"
 tokio-io-timeout = "1.2.0"
-reqwest = { version = "0.11.13", features = ["json"] }
-async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
+tokio-stream = { version = "0.1.11", features = ["fs"] }
+tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+toml = "0.7"
+trust-dns-resolver = "0.22"
+url = "2"
+uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
@@ -93,6 +102,7 @@ log = "0.4"
 pretty_env_logger = "0.4"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
+testdir = "0.7.2"
 tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 
 [workspace]
@@ -101,18 +111,14 @@ members = [
   "deltachat_derive",
   "deltachat-jsonrpc",
   "deltachat-rpc-server",
+  "deltachat-ratelimit",
+  "deltachat-repl",
   "format-flowed",
 ]
 
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
-required-features = ["repl"]
-
-[[example]]
-name = "repl"
-path = "examples/repl/main.rs"
-required-features = ["repl"]
 
 
 [[bench]]
@@ -139,14 +145,15 @@ harness = false
 name = "get_chatlist"
 harness = false
 
+[[bench]]
+name = "send_events"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = [
   "async-native-tls/vendored",
-  "async-smtp/native-tls-vendored",
   "rusqlite/bundled-sqlcipher-vendored-openssl",
   "reqwest/native-tls-vendored"
 ]
-nightly = ["pgp/nightly"]

README.md

@@ -19,10 +19,19 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
+Optionally, install `deltachat-repl` binary with
+```
+$ cargo install --path deltachat-repl/
+```
+and run as
+```
+$ deltachat-repl ~/deltachat-db
+```
+
 Configure your account (if not already configured):
 
 ```
@@ -104,7 +113,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests
@@ -161,7 +170,9 @@ Language bindings are available for:
   - over cffi (legacy): \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
   - over jsonrpc built with napi.rs: \[[📂 source](https://github.com/deltachat/napi-jsonrpc) | [📦 npm](https://www.npmjs.com/package/@deltachat/napi-jsonrpc)\]
 - **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go**[^1] \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
+- **Go**
+  - over jsonrpc: \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]
+  - over cffi[^1]: \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
 - **Free Pascal**[^1] \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
 - **Java** and **Swift** (contained in the Android/iOS repos)
 

benches/contacts.rs

@@ -14,7 +14,7 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
         .unwrap();
 
     let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .map(|i| format!("Name {i}\naddr{i}@example.org\n"))
         .collect::<Vec<String>>()
         .join("");
 

benches/create_account.rs

@@ -1,6 +1,7 @@
+use std::path::PathBuf;
+
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::accounts::Accounts;
-use std::path::PathBuf;
 use tempfile::tempdir;
 
 async fn create_accounts(n: u32) {

benches/get_chat_msgs.rs

@@ -1,7 +1,6 @@
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
 use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
@@ -15,7 +14,7 @@ async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
         .unwrap();
 
     for c in chats.iter().take(10) {
-        black_box(chat::get_chat_msgs(&context, *c, 0).await.ok());
+        black_box(chat::get_chat_msgs(&context, *c).await.ok());
     }
 }
 

benches/get_chatlist.rs

@@ -1,7 +1,6 @@
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
 use deltachat::stock_str::StockStrings;

benches/search_msgs.rs

@@ -1,8 +1,9 @@
+use std::path::Path;
+
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::context::Context;
 use deltachat::stock_str::StockStrings;
 use deltachat::Events;
-use std::path::Path;
 
 async fn search_benchmark(dbfile: impl AsRef<Path>) {
     let id = 100;

benches/send_events.rs

@@ -0,0 +1,47 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::{info, Event, EventType, Events};
+use tempfile::tempdir;
+
+async fn send_events_benchmark(context: &Context) {
+    let emitter = context.get_event_emitter();
+    for _i in 0..1_000_000 {
+        info!(context, "interesting event...");
+    }
+    info!(context, "DONE");
+
+    loop {
+        match emitter.recv().await.unwrap() {
+            Event {
+                typ: EventType::Info(info),
+                ..
+            } if info.contains("DONE") => {
+                break;
+            }
+            _ => {}
+        }
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    let context = rt.block_on(async {
+        Context::new(&dbfile, 100, Events::new(), StockStrings::new())
+            .await
+            .expect("failed to create context")
+    });
+    let executor = tokio::runtime::Runtime::new().unwrap();
+
+    c.bench_function("Sending 1.000.000 events", |b| {
+        b.to_async(&executor)
+            .iter(|| send_events_benchmark(&context))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.105.0"
+version = "1.111.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -17,7 +17,7 @@ crate-type = ["cdylib", "staticlib"]
 deltachat = { path = "../", default-features = false }
 deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
 libc = "0.2"
-human-panic = "1"
+human-panic = { version = "1", default-features = false }
 num-traits = "0.2"
 serde_json = "1.0"
 tokio = { version = "1", features = ["rt-multi-thread"] }
@@ -29,6 +29,5 @@ once_cell = "1.17.0"
 [features]
 default = ["vendored"]
 vendored = ["deltachat/vendored"]
-nightly = ["deltachat/nightly"]
-jsonrpc = ["deltachat-jsonrpc"]
+jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/Doxyfile.css

@@ -1,14 +1,24 @@
 
+:root {
+	--accent: hsl(0 0% 85%);
+}
+
+@media (prefers-color-scheme: dark) {
+	:root {
+		--accent: hsl(0 0% 25%);
+	}
+}
+
 /* the code snippet frame, defaults to white which tends to get badly readable in combination with explaining text around */
 div.fragment {
-	background-color: #e0e0e0;
+	background-color: var(--accent);
 	border: 0;
 	padding: 1em;
 	border-radius: 6px;
 }
 
 code {
-	background-color: #e0e0e0;
+	background-color: var(--accent);
 	padding-left: .5em;
 	padding-right: .5em;
 	border-radius: 6px;

deltachat-ffi/deltachat.h

@@ -24,6 +24,7 @@ typedef struct _dc_provider  dc_provider_t;
 typedef struct _dc_event     dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
+typedef struct _dc_backup_provider dc_backup_provider_t;
 
 // Alias for backwards compatibility, use dc_event_emitter_t instead.
 typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
@@ -612,7 +613,7 @@ int             dc_all_work_done             (dc_context_t* context);
  * While dc_configure() returns immediately,
  * the started configuration-job may take a while.
  *
- * During configuration, #DC_EVENT_CONFIGURE_PROGRESS events are emmited;
+ * During configuration, #DC_EVENT_CONFIGURE_PROGRESS events are emitted;
  * they indicate a successful configuration as well as errors
  * and may be used to create a progress bar.
  *
@@ -869,7 +870,7 @@ uint32_t        dc_get_chat_id_by_contact_id (dc_context_t* context, uint32_t co
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to send the message to.
  * @param msg The message object to send to the chat defined by the chat ID.
- *     On succcess, msg_id and state of the object are set up,
+ *     On success, msg_id and state of the object are set up,
  *     The function does not take ownership of the object,
  *     so you have to free it using dc_msg_unref() as usual.
  * @return The ID of the message that is being prepared.
@@ -880,7 +881,7 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
 /**
  * Send a message defined by a dc_msg_t object to a chat.
  *
- * Sends the event #DC_EVENT_MSGS_CHANGED on succcess.
+ * Sends the event #DC_EVENT_MSGS_CHANGED on success.
  * However, this does not imply, the message really reached the recipient -
  * sending may be delayed e.g. due to network problems. However, from your
  * view, you're done with the message. Sooner or later it will find its way.
@@ -909,7 +910,7 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * @param chat_id The chat ID to send the message to.
  *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
- *     On succcess, msg_id of the object is set up,
+ *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
  *     so you have to free it using dc_msg_unref() as usual.
  * @return The ID of the message that is about to be sent. 0 in case of errors.
@@ -926,7 +927,7 @@ uint32_t        dc_send_msg                  (dc_context_t* context, uint32_t ch
  * @param chat_id The chat ID to send the message to.
  *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
- *     On succcess, msg_id of the object is set up,
+ *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
  *     so you have to free it using dc_msg_unref() as usual.
  * @return The ID of the message that is about to be sent. 0 in case of errors.
@@ -937,7 +938,7 @@ uint32_t        dc_send_msg_sync                  (dc_context_t* context, uint32
 /**
  * Send a simple text message a given chat.
  *
- * Sends the event #DC_EVENT_MSGS_CHANGED on succcess.
+ * Sends the event #DC_EVENT_MSGS_CHANGED on success.
  * However, this does not imply, the message really reached the recipient -
  * sending may be delayed e.g. due to network problems. However, from your
  * view, you're done with the message. Sooner or later it will find its way.
@@ -1146,7 +1147,7 @@ void            dc_set_draft                 (dc_context_t* context, uint32_t ch
  *     // not now and not when this code is executed again
  *     dc_add_device_msg(context, "update-123", NULL);
  * } else {
- *     // welcome message was not added now, this is an oder installation,
+ *     // welcome message was not added now, this is an older installation,
  *     // add a changelog
  *     dc_add_device_msg(context, "update-123", changelog_msg);
  * }
@@ -1159,7 +1160,7 @@ uint32_t        dc_add_device_msg            (dc_context_t* context, const char*
 
 /**
  * Check if a device-message with a given label was ever added.
- * Device-messages can be added dc_add_device_msg().
+ * Device-messages can be added with dc_add_device_msg().
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -2100,8 +2101,7 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
 
 /**
  * Import/export things.
- * During backup import/export IO must not be started,
- * if needed stop IO using dc_accounts_stop_io() or dc_stop_io() first.
+ *
  * What to do is defined by the _what_ parameter which may be one of the following:
  *
  * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`
@@ -2295,6 +2295,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_MISMATCH           220 // id=contact
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
+#define         DC_QR_BACKUP                 251
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
@@ -2320,7 +2321,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask whether to verify the contact;
  *   if so, start the protocol with dc_join_securejoin().
  *
- * - DC_QR_ASK_VERIFYGROUP withdc_lot_t::text1=Group name:
+ * - DC_QR_ASK_VERIFYGROUP with dc_lot_t::text1=Group name:
  *   ask whether to join the group;
  *   if so, start the protocol with dc_join_securejoin().
  *
@@ -2340,6 +2341,10 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask the user if they want to create an account on the given domain,
  *   if so, call dc_set_config_from_qr() and then dc_configure().
  *
+ * - DC_QR_BACKUP:
+ *   ask the user if they want to set up a new device.
+ *   If so, pass the qr-code to dc_receive_backup().
+ *
  * - DC_QR_WEBRTC_INSTANCE with dc_lot_t::text1=domain:
  *   ask the user if they want to use the given service for video chats;
  *   if so, call dc_set_config_from_qr().
@@ -2630,6 +2635,117 @@ char* dc_get_last_error (dc_context_t* context);
 void dc_str_unref (char* str);
 
 
+/**
+ * @class dc_backup_provider_t
+ *
+ * Set up another device.
+ */
+
+/**
+ * Creates an object for sending a backup to another device.
+ *
+ * The backup is sent to through a peer-to-peer channel which is bootstrapped
+ * by a QR-code.  The backup contains the entire state of the account
+ * including credentials.  This can be used to setup a new device.
+ *
+ * This is a blocking call as some preparations are made like e.g. exporting
+ * the database.  Once this function returns, the backup is being offered to
+ * remote devices.  To wait until one device received the backup, use
+ * dc_backup_provider_wait().  Alternatively abort the operation using
+ * dc_stop_ongoing_process().
+ *
+ * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate
+ * state and progress.
+ *
+ * @memberof dc_backup_provider_t
+ * @param context The context.
+ * @return Opaque object for sending the backup.
+ *    On errors, NULL is returned and dc_get_last_error() returns an error that
+ *    should be shown to the user.
+ */
+dc_backup_provider_t* dc_backup_provider_new (dc_context_t* context);
+
+
+/**
+ * Returns the QR code text that will offer the backup to other devices.
+ *
+ * The QR code contains a ticket which will validate the backup and provide
+ * authentication for both the provider and the recipient.
+ *
+ * The scanning device should call the scanned text to dc_check_qr().  If
+ * dc_check_qr() returns DC_QR_BACKUP, the backup transfer can be started using
+ * dc_get_backup().
+ *
+ * @memberof dc_backup_provider_t
+ * @param backup_provider The backup provider object as created by
+ *    dc_backup_provider_new().
+ * @return The text that should be put in the QR code.
+ *    On errors an empty string is returned, NULL is never returned.
+ *    the returned string must be released using dc_str_unref() after usage.
+ */
+char* dc_backup_provider_get_qr (const dc_backup_provider_t* backup_provider);
+
+
+/**
+ * Returns the QR code SVG image that will offer the backup to other devices.
+ *
+ * This works like dc_backup_provider_qr() but returns the text of a rendered
+ * SVG image containing the QR code.
+ *
+ * @memberof dc_backup_provider_t
+ * @param backup_provider The backup provider object as created by
+ *    dc_backup_provider_new().
+ * @return The QR code rendered as SVG.
+ *    On errors an empty string is returned, NULL is never returned.
+ *    the returned string must be released using dc_str_unref() after usage.
+ */
+char* dc_backup_provider_get_qr_svg (const dc_backup_provider_t* backup_provider);
+
+/**
+ * Waits for the sending to finish.
+ *
+ * This is a blocking call and should only be called once.
+ *
+ * @memberof dc_backup_provider_t
+ * @param backup_provider The backup provider object as created by
+ *    dc_backup_provider_new().  If NULL is given nothing is done.
+ */
+void dc_backup_provider_wait (dc_backup_provider_t* backup_provider);
+
+/**
+ * Frees a dc_backup_provider_t object.
+ *
+ * @memberof dc_backup_provider_t
+ * @param backup_provider The backup provider object as created by
+ *    dc_backup_provider_new().
+ */
+void dc_backup_provider_unref (dc_backup_provider_t* backup_provider);
+
+/**
+ * Gets a backup offered by a dc_backup_provider_t object on another device.
+ *
+ * This function is called on a device that scanned the QR code offered by
+ * dc_backup_sender_qr() or dc_backup_sender_qr_svg().  Typically this is a
+ * different device than that which provides the backup.
+ *
+ * This call will block while the backup is being transferred and only
+ * complete on success or failure.  Use dc_stop_ongoing_process() to abort it
+ * early.
+ *
+ * During execution of the job #DC_EVENT_IMEX_PROGRESS is sent out to indicate
+ * state and progress.  The process is finished when the event emits either 0
+ * or 1000, 0 means it failed and 1000 means it succeeded.  These events are
+ * for showing progress and informational only, success and failure is also
+ * shown in the return code of this function.
+ *
+ * @memberof dc_context_t
+ * @param context The context.
+ * @param qr The qr code text, dc_check_qr() must have returned DC_QR_BACKUP
+ *    on this text.
+ * @return 0=failure, 1=success.
+ */
+int dc_receive_backup (dc_context_t* context, const char* qr);
+
 /**
  * @class dc_accounts_t
  *
@@ -3184,7 +3300,7 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
  * it takes the chat ID and the message ID as returned by dc_chatlist_get_chat_id() and dc_chatlist_get_msg_id()
  * as arguments. The chatlist object itself is not needed directly.
  *
- * This maybe useful if you convert the complete object into a different represenation
+ * This maybe useful if you convert the complete object into a different representation
  * as done e.g. in the node-bindings.
  * If you have access to the chatlist object in some way, using this function is not recommended,
  * use dc_chatlist_get_summary() in this case instead.
@@ -4327,6 +4443,18 @@ void            dc_msg_set_text               (dc_msg_t* msg, const char* text);
 void            dc_msg_set_html               (dc_msg_t* msg, const char* html);
 
 
+/**
+ * Sets the email's subject. If it's empty, a default subject
+ * will be used (e.g. `Message from Alice` or `Re: <last subject>`).
+ * This does not alter any information in the database.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @param subject The new subject.
+ */
+void            dc_msg_set_subject            (dc_msg_t* msg, const char* subject);
+
+
 /**
  * Set different sender name for a message.
  * This overrides the name set by the dc_set_config()-option `displayname`.
@@ -5671,7 +5799,7 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_INCOMING_MSG             2005
 
 /**
- * Downloading a bunch of messages just finished. This is an experimental
+ * Downloading a bunch of messages just finished. This is an
  * event to allow the UI to only show one notification per message bunch,
  * instead of cluttering the user with many notifications.
  * For each of the msg_ids, an additional #DC_EVENT_INCOMING_MSG event was emitted before.
@@ -5811,7 +5939,7 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     300=vg-/vc-request received, typically shown as "bob@addr joins".
  *     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
- *     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
+ *     800=contact added to chat, shown as "bob@addr securely joined GROUP". Only for the verified-group-protocol.
  *     1000=Protocol finished for this contact.
  */
 #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS      2060
@@ -6329,7 +6457,7 @@ void dc_event_unref(dc_event_t* event);
 ///
 /// Used in status messages.
 ///
-/// @deperecated 2022-09-10
+/// @deprecated 2022-09-10
 #define DC_STR_EPHEMERAL_MINUTE           77
 
 /// "Message deletion timer is set to 1 hour."
@@ -6611,7 +6739,7 @@ void dc_event_unref(dc_event_t* event);
 
 /// "You changed your email address from %1$s to %2$s.
 /// If you now send a message to a group, contacts there will automatically
-/// replace the old with your new address.\n\nIt's highly advised to set up 
+/// replace the old with your new address.\n\n It's highly advised to set up 
 /// your old email provider to forward all emails to your new email address. 
 /// Otherwise you might miss messages of contacts who did not get your new 
 /// address yet." + the link to the AEAP blog post

deltachat-ffi/src/lib.rs

@@ -23,35 +23,36 @@ use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use num_traits::{FromPrimitive, ToPrimitive};
-use once_cell::sync::Lazy;
-use rand::Rng;
-use tokio::runtime::Runtime;
-use tokio::sync::RwLock;
-
-use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus};
+use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, ContactId, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
+use deltachat::imex::BackupProvider;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
+use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
 use deltachat::stock_str::StockStrings;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
+use num_traits::{FromPrimitive, ToPrimitive};
+use once_cell::sync::Lazy;
+use rand::Rng;
+use tokio::runtime::Runtime;
+use tokio::sync::RwLock;
 use tokio::task::JoinHandle;
 
 mod dc_array;
 mod lot;
 
 mod string;
-use self::string::*;
 use deltachat::chatlist::Chatlist;
 
+use self::string::*;
+
 // as C lacks a good and portable error handling,
 // in general, the C Interface is forgiving wrt to bad parameters.
 // - objects returned by some functions
@@ -60,7 +61,8 @@ use deltachat::chatlist::Chatlist;
 //   this avoids panics if the ui just forgets to handle a case
 // - finally, this behaviour matches the old core-c API and UIs already depend on it
 
-// TODO: constants
+const DC_GCM_ADDDAYMARKER: u32 = 0x01;
+const DC_GCM_INFO_ONLY: u32 = 0x02;
 
 // dc_context_t
 
@@ -115,7 +117,7 @@ pub unsafe extern "C" fn dc_context_new(
     match ctx {
         Ok(ctx) => Box::into_raw(Box::new(ctx)),
         Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
+            eprintln!("failed to create context: {err:#}");
             ptr::null_mut()
         }
     }
@@ -139,7 +141,7 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     )) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
-            eprintln!("failed to create context: {:#}", err);
+            eprintln!("failed to create context: {err:#}");
             ptr::null_mut()
         }
     }
@@ -214,7 +216,7 @@ pub unsafe extern "C" fn dc_set_config(
         if key.starts_with("ui.") {
             ctx.set_ui_config(&key, value.as_deref())
                 .await
-                .with_context(|| format!("Can't set {} to {:?}", key, value))
+                .with_context(|| format!("Can't set {key} to {value:?}"))
                 .log_err(ctx, "dc_set_config() failed")
                 .is_ok() as libc::c_int
         } else {
@@ -222,7 +224,7 @@ pub unsafe extern "C" fn dc_set_config(
                 Ok(key) => ctx
                     .set_config(key, value.as_deref())
                     .await
-                    .with_context(|| format!("Can't set {} to {:?}", key, value))
+                    .with_context(|| format!("Can't set {key} to {value:?}"))
                     .log_err(ctx, "dc_set_config() failed")
                     .is_ok() as libc::c_int,
                 Err(_) => {
@@ -291,12 +293,12 @@ pub unsafe extern "C" fn dc_set_stock_translation(
             Some(id) => match ctx.set_stock_translation(id, msg).await {
                 Ok(()) => 1,
                 Err(err) => {
-                    warn!(ctx, "set_stock_translation failed: {}", err);
+                    warn!(ctx, "set_stock_translation failed: {err:#}");
                     0
                 }
             },
             None => {
-                warn!(ctx, "invalid stock message id {}", stock_id);
+                warn!(ctx, "invalid stock message id {stock_id}");
                 0
             }
         }
@@ -319,7 +321,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr(
         match qr::set_config_from_qr(ctx, &qr).await {
             Ok(()) => 1,
             Err(err) => {
-                error!(ctx, "Failed to create account from QR code: {}", err);
+                error!(ctx, "Failed to create account from QR code: {err:#}");
                 0
             }
         }
@@ -337,7 +339,7 @@ pub unsafe extern "C" fn dc_get_info(context: *const dc_context_t) -> *mut libc:
         match ctx.get_info().await {
             Ok(info) => render_info(info).unwrap_or_default().strdup(),
             Err(err) => {
-                warn!(ctx, "failed to get info: {}", err);
+                warn!(ctx, "failed to get info: {err:#}");
                 "".strdup()
             }
         }
@@ -349,7 +351,7 @@ fn render_info(
 ) -> std::result::Result<String, std::fmt::Error> {
     let mut res = String::new();
     for (key, value) in &info {
-        writeln!(&mut res, "{}={}", key, value)?;
+        writeln!(&mut res, "{key}={value}")?;
     }
 
     Ok(res)
@@ -378,7 +380,7 @@ pub unsafe extern "C" fn dc_get_connectivity_html(
         match ctx.get_connectivity_html().await {
             Ok(html) => html.strdup(),
             Err(err) => {
-                error!(ctx, "Failed to get connectivity html: {}", err);
+                error!(ctx, "Failed to get connectivity html: {err:#}");
                 "".strdup()
             }
         }
@@ -420,6 +422,10 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
     })
 }
 
+fn spawn_configure(ctx: Context) {
+    spawn(async move { ctx.configure().await.log_err(&ctx, "Configure failed") });
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
     if context.is_null() {
@@ -428,8 +434,7 @@ pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
     }
 
     let ctx = &*context;
-
-    spawn(async move { ctx.configure().await.log_err(ctx, "Configure failed") });
+    spawn_configure(ctx.clone());
 }
 
 #[no_mangle]
@@ -1136,7 +1141,7 @@ pub unsafe extern "C" fn dc_get_draft(context: *mut dc_context_t, chat_id: u32)
             }
             Ok(None) => ptr::null_mut(),
             Err(err) => {
-                error!(ctx, "Failed to get draft for chat #{}: {}", chat_id, err);
+                error!(ctx, "Failed to get draft for chat #{chat_id}: {err:#}");
                 ptr::null_mut()
             }
         }
@@ -1156,12 +1161,21 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     }
     let ctx = &*context;
 
+    let info_only = (flags & DC_GCM_INFO_ONLY) != 0;
+    let add_daymarker = (flags & DC_GCM_ADDDAYMARKER) != 0;
     block_on(async move {
         Box::into_raw(Box::new(
-            chat::get_chat_msgs(ctx, ChatId::new(chat_id), flags)
-                .await
-                .unwrap_or_log_default(ctx, "failed to get chat msgs")
-                .into(),
+            chat::get_chat_msgs_ex(
+                ctx,
+                ChatId::new(chat_id),
+                MessageListOptions {
+                    info_only,
+                    add_daymarker,
+                },
+            )
+            .await
+            .unwrap_or_log_default(ctx, "failed to get chat msgs")
+            .into(),
         ))
     })
 }
@@ -1285,11 +1299,11 @@ pub unsafe extern "C" fn dc_get_chat_media(
     } else {
         Some(ChatId::new(chat_id))
     };
-    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {}", msg_type));
+    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {msg_type}"));
     let or_msg_type2 =
-        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {}", or_msg_type2));
+        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {or_msg_type2}"));
     let or_msg_type3 =
-        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
+        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {or_msg_type3}"));
 
     block_on(async move {
         Box::into_raw(Box::new(
@@ -1321,11 +1335,11 @@ pub unsafe extern "C" fn dc_get_next_media(
     };
 
     let ctx = &*context;
-    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {}", msg_type));
+    let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {msg_type}"));
     let or_msg_type2 =
-        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {}", or_msg_type2));
+        from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {or_msg_type2}"));
     let or_msg_type3 =
-        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
+        from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {or_msg_type3}"));
 
     block_on(async move {
         chat::get_next_media(
@@ -1717,7 +1731,7 @@ pub unsafe extern "C" fn dc_get_chat_encrinfo(
             .await
             .map(|s| s.strdup())
             .unwrap_or_else(|e| {
-                error!(ctx, "{}", e);
+                error!(ctx, "{e:#}");
                 ptr::null_mut()
             })
     })
@@ -1880,7 +1894,7 @@ pub unsafe extern "C" fn dc_resend_msgs(
     let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
 
     if let Err(err) = block_on(chat::resend_msgs(ctx, &msg_ids)) {
-        error!(ctx, "Resending failed: {}", err);
+        error!(ctx, "Resending failed: {err:#}");
         0
     } else {
         1
@@ -1921,14 +1935,11 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
                     // C-core API returns empty messages, do the same
                     warn!(
                         ctx,
-                        "dc_get_msg called with special msg_id={}, returning empty msg", msg_id
+                        "dc_get_msg called with special msg_id={msg_id}, returning empty msg"
                     );
                     message::Message::default()
                 } else {
-                    error!(
-                        ctx,
-                        "dc_get_msg could not retrieve msg_id {}: {}", msg_id, e
-                    );
+                    error!(ctx, "dc_get_msg could not retrieve msg_id {msg_id}: {e:#}");
                     return ptr::null_mut();
                 }
             }
@@ -1993,12 +2004,10 @@ pub unsafe extern "C" fn dc_create_contact(
     let ctx = &*context;
     let name = to_string_lossy(name);
 
-    block_on(async move {
-        Contact::create(ctx, &name, &to_string_lossy(addr))
-            .await
-            .map(|id| id.to_u32())
-            .unwrap_or(0)
-    })
+    block_on(Contact::create(ctx, &name, &to_string_lossy(addr)))
+        .log_err(ctx, "Cannot create contact")
+        .map(|id| id.to_u32())
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -2123,7 +2132,7 @@ pub unsafe extern "C" fn dc_get_contact_encrinfo(
             .await
             .map(|s| s.strdup())
             .unwrap_or_else(|e| {
-                error!(ctx, "{}", e);
+                error!(ctx, "{e:#}");
                 ptr::null_mut()
             })
     })
@@ -2145,7 +2154,7 @@ pub unsafe extern "C" fn dc_delete_contact(
         match Contact::delete(ctx, contact_id).await {
             Ok(_) => 1,
             Err(err) => {
-                error!(ctx, "cannot delete contact: {}", err);
+                error!(ctx, "cannot delete contact: {err:#}");
                 0
             }
         }
@@ -2171,6 +2180,14 @@ pub unsafe extern "C" fn dc_get_contact(
     })
 }
 
+fn spawn_imex(ctx: Context, what: imex::ImexMode, param1: String, passphrase: Option<String>) {
+    spawn(async move {
+        imex::imex(&ctx, what, param1.as_ref(), passphrase)
+            .await
+            .log_err(&ctx, "IMEX failed")
+    });
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_imex(
     context: *mut dc_context_t,
@@ -2185,7 +2202,7 @@ pub unsafe extern "C" fn dc_imex(
     let what = match imex::ImexMode::from_i32(what_raw) {
         Some(what) => what,
         None => {
-            eprintln!("ignoring invalid argument {} to dc_imex", what_raw);
+            eprintln!("ignoring invalid argument {what_raw} to dc_imex");
             return;
         }
     };
@@ -2194,11 +2211,7 @@ pub unsafe extern "C" fn dc_imex(
     let ctx = &*context;
 
     if let Some(param1) = to_opt_string_lossy(param1) {
-        spawn(async move {
-            imex::imex(ctx, what, param1.as_ref(), passphrase)
-                .await
-                .log_err(ctx, "IMEX failed")
-        });
+        spawn_imex(ctx.clone(), what, param1, passphrase);
     } else {
         eprintln!("dc_imex called without a valid directory");
     }
@@ -2221,7 +2234,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
             Err(err) => {
                 // do not bubble up error to the user,
                 // the ui will expect that the file does not exist or cannot be accessed
-                warn!(ctx, "dc_imex_has_backup: {}", err);
+                warn!(ctx, "dc_imex_has_backup: {err:#}");
                 ptr::null_mut()
             }
         }
@@ -2240,7 +2253,7 @@ pub unsafe extern "C" fn dc_initiate_key_transfer(context: *mut dc_context_t) ->
         match imex::initiate_key_transfer(ctx).await {
             Ok(res) => res.strdup(),
             Err(err) => {
-                error!(ctx, "dc_initiate_key_transfer(): {}", err);
+                error!(ctx, "dc_initiate_key_transfer(): {err:#}");
                 ptr::null_mut()
             }
         }
@@ -2265,7 +2278,7 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
         {
             Ok(()) => 1,
             Err(err) => {
-                warn!(ctx, "dc_continue_key_transfer: {}", err);
+                warn!(ctx, "dc_continue_key_transfer: {err:#}");
                 0
             }
         }
@@ -2653,7 +2666,7 @@ pub unsafe fn dc_array_is_independent(
 ///
 /// This is the structure behind [dc_chatlist_t] which is the opaque
 /// structure representing a chatlist in the FFI API.  It exists
-/// because the FFI API has a refernce from the message to the
+/// because the FFI API has a reference from the message to the
 /// context, but the Rust API does not, so the FFI layer needs to glue
 /// these together.
 pub struct ChatlistWrapper {
@@ -2696,7 +2709,7 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
     match ffi_list.list.get_chat_id(index) {
         Ok(chat_id) => chat_id.to_u32(),
         Err(err) => {
-            warn!(ctx, "get_chat_id failed: {}", err);
+            warn!(ctx, "get_chat_id failed: {err:#}");
             0
         }
     }
@@ -2716,7 +2729,7 @@ pub unsafe extern "C" fn dc_chatlist_get_msg_id(
     match ffi_list.list.get_msg_id(index) {
         Ok(msg_id) => msg_id.map_or(0, |msg_id| msg_id.to_u32()),
         Err(err) => {
-            warn!(ctx, "get_msg_id failed: {}", err);
+            warn!(ctx, "get_msg_id failed: {err:#}");
             0
         }
     }
@@ -2797,7 +2810,7 @@ pub unsafe extern "C" fn dc_chatlist_get_context(
 ///
 /// This is the structure behind [dc_chat_t] which is the opaque
 /// structure representing a chat in the FFI API.  It exists
-/// because the FFI API has a refernce from the message to the
+/// because the FFI API has a reference from the message to the
 /// context, but the Rust API does not, so the FFI layer needs to glue
 /// these together.
 pub struct ChatWrapper {
@@ -2875,7 +2888,7 @@ pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut
             Ok(Some(p)) => p.to_string_lossy().strdup(),
             Ok(None) => ptr::null_mut(),
             Err(err) => {
-                error!(ctx, "failed to get profile image: {:?}", err);
+                error!(ctx, "failed to get profile image: {err:#}");
                 ptr::null_mut()
             }
         }
@@ -3027,7 +3040,7 @@ pub unsafe extern "C" fn dc_chat_get_info_json(
         let chat = match chat::Chat::load_from_db(ctx, ChatId::new(chat_id)).await {
             Ok(chat) => chat,
             Err(err) => {
-                error!(ctx, "dc_get_chat_info_json() failed to load chat: {}", err);
+                error!(ctx, "dc_get_chat_info_json() failed to load chat: {err:#}");
                 return "".strdup();
             }
         };
@@ -3036,7 +3049,7 @@ pub unsafe extern "C" fn dc_chat_get_info_json(
             Err(err) => {
                 error!(
                     ctx,
-                    "dc_get_chat_info_json() failed to get chat info: {}", err
+                    "dc_get_chat_info_json() failed to get chat info: {err:#}"
                 );
                 return "".strdup();
             }
@@ -3053,7 +3066,7 @@ pub unsafe extern "C" fn dc_chat_get_info_json(
 ///
 /// This is the structure behind [dc_msg_t] which is the opaque
 /// structure representing a message in the FFI API.  It exists
-/// because the FFI API has a refernce from the message to the
+/// because the FFI API has a reference from the message to the
 /// context, but the Rust API does not, so the FFI layer needs to glue
 /// these together.
 pub struct MessageWrapper {
@@ -3073,7 +3086,7 @@ pub unsafe extern "C" fn dc_msg_new(
         return ptr::null_mut();
     }
     let context = &*context;
-    let viewtype = from_prim(viewtype).expect(&format!("invalid viewtype = {}", viewtype));
+    let viewtype = from_prim(viewtype).expect(&format!("invalid viewtype = {viewtype}"));
     let msg = MessageWrapper {
         context,
         message: message::Message::new(viewtype),
@@ -3256,7 +3269,7 @@ pub unsafe extern "C" fn dc_msg_get_webxdc_blob(
             ptr as *mut libc::c_char
         }
         Err(err) => {
-            eprintln!("failed read blob from archive: {}", err);
+            eprintln!("failed read blob from archive: {err}");
             ptr::null_mut()
         }
     }
@@ -3275,7 +3288,7 @@ pub unsafe extern "C" fn dc_msg_get_webxdc_info(msg: *mut dc_msg_t) -> *mut libc
         let info = match ffi_msg.message.get_webxdc_info(ctx).await {
             Ok(info) => info,
             Err(err) => {
-                error!(ctx, "dc_msg_get_webxdc_info() failed to get info: {}", err);
+                error!(ctx, "dc_msg_get_webxdc_info() failed to get info: {err:#}");
                 return "".strdup();
             }
         };
@@ -3588,6 +3601,16 @@ pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::
     ffi_msg.message.set_html(to_opt_string_lossy(html))
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_subject(msg: *mut dc_msg_t, subject: *const libc::c_char) {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_subject()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    ffi_msg.message.set_subject(to_string_lossy(subject));
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_override_sender_name(
     msg: *mut dc_msg_t,
@@ -3796,7 +3819,7 @@ pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
 ///
 /// This is the structure behind [dc_contact_t] which is the opaque
 /// structure representing a contact in the FFI API.  It exists
-/// because the FFI API has a refernce from the message to the
+/// because the FFI API has a reference from the message to the
 /// context, but the Rust API does not, so the FFI layer needs to glue
 /// these together.
 pub struct ContactWrapper {
@@ -3975,13 +3998,10 @@ pub unsafe extern "C" fn dc_contact_get_verifier_addr(
     }
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
-    block_on(Contact::get_verifier_addr(
-        ctx,
-        &ffi_contact.contact.get_id(),
-    ))
-    .log_err(ctx, "failed to get verifier for contact")
-    .unwrap_or_default()
-    .strdup()
+    block_on(ffi_contact.contact.get_verifier_addr(ctx))
+        .log_err(ctx, "failed to get verifier for contact")
+        .unwrap_or_default()
+        .strdup()
 }
 
 #[no_mangle]
@@ -3992,12 +4012,12 @@ pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t)
     }
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
-    let contact_id = block_on(Contact::get_verifier_id(ctx, &ffi_contact.contact.get_id()))
+    let verifier_contact_id = block_on(ffi_contact.contact.get_verifier_id(ctx))
         .log_err(ctx, "failed to get verifier")
         .unwrap_or_default()
         .unwrap_or_default();
 
-    contact_id.to_u32()
+    verifier_contact_id.to_u32()
 }
 // dc_lot_t
 
@@ -4123,6 +4143,116 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
 }
 
+pub struct BackupProviderWrapper {
+    context: *const dc_context_t,
+    provider: BackupProvider,
+}
+
+pub type dc_backup_provider_t = BackupProviderWrapper;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_backup_provider_new(
+    context: *mut dc_context_t,
+) -> *mut dc_backup_provider_t {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_backup_provider_new()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+    block_on(BackupProvider::prepare(ctx))
+        .map(|provider| BackupProviderWrapper {
+            context: ctx,
+            provider,
+        })
+        .map(|ffi_provider| Box::into_raw(Box::new(ffi_provider)))
+        .log_err(ctx, "BackupProvider failed")
+        .context("BackupProvider failed")
+        .set_last_error(ctx)
+        .unwrap_or(ptr::null_mut())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_backup_provider_get_qr(
+    provider: *const dc_backup_provider_t,
+) -> *mut libc::c_char {
+    if provider.is_null() {
+        eprintln!("ignoring careless call to dc_backup_provider_qr");
+        return "".strdup();
+    }
+    let ffi_provider = &*provider;
+    let ctx = &*ffi_provider.context;
+    deltachat::qr::format_backup(&ffi_provider.provider.qr())
+        .log_err(ctx, "BackupProvider get_qr failed")
+        .context("BackupProvider get_qr failed")
+        .set_last_error(ctx)
+        .unwrap_or_default()
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_backup_provider_get_qr_svg(
+    provider: *const dc_backup_provider_t,
+) -> *mut libc::c_char {
+    if provider.is_null() {
+        eprintln!("ignoring careless call to dc_backup_provider_qr_svg()");
+        return "".strdup();
+    }
+    let ffi_provider = &*provider;
+    let ctx = &*ffi_provider.context;
+    let provider = &ffi_provider.provider;
+    block_on(generate_backup_qr(ctx, &provider.qr()))
+        .log_err(ctx, "BackupProvider get_qr_svg failed")
+        .context("BackupProvider get_qr_svg failed")
+        .set_last_error(ctx)
+        .unwrap_or_default()
+        .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provider_t) {
+    if provider.is_null() {
+        eprintln!("ignoring careless call to dc_backup_provider_wait()");
+        return;
+    }
+    let ffi_provider = &mut *provider;
+    let ctx = &*ffi_provider.context;
+    let provider = &mut ffi_provider.provider;
+    block_on(provider)
+        .log_err(ctx, "Failed to await BackupProvider")
+        .context("Failed to await BackupProvider")
+        .set_last_error(ctx)
+        .ok();
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_backup_provider_unref(provider: *mut dc_backup_provider_t) {
+    drop(Box::from_raw(provider));
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_receive_backup(
+    context: *mut dc_context_t,
+    qr: *const libc::c_char,
+) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_receive_backup()");
+        return 0;
+    }
+    let ctx = &*context;
+    let qr_text = to_string_lossy(qr);
+    let qr = match block_on(qr::check_qr(ctx, &qr_text)).log_err(ctx, "Invalid QR code") {
+        Ok(qr) => qr,
+        Err(_) => return 0,
+    };
+    spawn(async move {
+        imex::get_backup(ctx, qr)
+            .await
+            .log_err(ctx, "Get backup failed")
+            .ok();
+    });
+    1
+}
+
 trait ResultExt<T, E> {
     /// Like `log_err()`, but:
     /// - returns the default value instead of an Err value.
@@ -4136,13 +4266,63 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
         match self {
             Ok(t) => t,
             Err(err) => {
-                error!(context, "{}: {}", message, err);
+                error!(context, "{message}: {err:#}");
                 Default::default()
             }
         }
     }
 }
 
+trait ResultLastError<T, E>
+where
+    E: std::fmt::Display,
+{
+    /// Sets this `Err` value using [`Context::set_last_error`].
+    ///
+    /// Normally each FFI-API *should* call this if it handles an error from the Rust API:
+    /// errors which need to be reported to users in response to an API call need to be
+    /// propagated up the Rust API and at the FFI boundary need to be stored into the "last
+    /// error" so the FFI users can retrieve an appropriate error message on failure.  Often
+    /// you will want to combine this with a call to [`LogExt::log_err`].
+    ///
+    /// Since historically calls to the `deltachat::log::error!()` macro were (and sometimes
+    /// still are) shown as error toasts to the user, this macro also calls
+    /// [`Context::set_last_error`].  It is preferable however to rely on normal error
+    /// propagation in Rust code however and only use this `ResultExt::set_last_error` call
+    /// in the FFI layer.
+    ///
+    /// # Example
+    ///
+    /// Fully handling an error in the FFI code looks like this currently:
+    ///
+    /// ```no_compile
+    /// some_dc_rust_api_call_returning_result()
+    ///     .log_err(&context, "My API call failed")
+    ///     .context("My API call failed")
+    ///     .set_last_error(&context)
+    ///     .unwrap_or_default()
+    /// ```
+    ///
+    /// As shows it is a shame the `.log_err()` call currently needs a message instead of
+    /// relying on an implicit call to the [`anyhow::Context`] call if needed.  This stems
+    /// from a time before we fully embraced anyhow.  Some day we'll also fix that.
+    ///
+    /// [`Context::set_last_error`]: context::Context::set_last_error
+    fn set_last_error(self, context: &context::Context) -> Result<T, E>;
+}
+
+impl<T, E> ResultLastError<T, E> for Result<T, E>
+where
+    E: std::fmt::Display,
+{
+    fn set_last_error(self, context: &context::Context) -> Result<T, E> {
+        if let Err(ref err) = self {
+            context.set_last_error(&format!("{err:#}"));
+        }
+        self
+    }
+}
+
 trait ResultNullableExt<T> {
     fn into_raw(self) -> *mut T;
 }
@@ -4312,7 +4492,7 @@ pub unsafe extern "C" fn dc_accounts_new(
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
         Err(err) => {
             // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
-            eprintln!("failed to create accounts: {:#}", err);
+            eprintln!("failed to create accounts: {err:#}");
             ptr::null_mut()
         }
     }
@@ -4380,8 +4560,7 @@ pub unsafe extern "C" fn dc_accounts_select_account(
             Ok(()) => 1,
             Err(err) => {
                 accounts.emit_event(EventType::Error(format!(
-                    "Failed to select account: {:#}",
-                    err
+                    "Failed to select account: {err:#}"
                 )));
                 0
             }
@@ -4403,10 +4582,7 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) -
         match accounts.add_account().await {
             Ok(id) => id,
             Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to add account: {:#}",
-                    err
-                )));
+                accounts.emit_event(EventType::Error(format!("Failed to add account: {err:#}")));
                 0
             }
         }
@@ -4427,10 +4603,7 @@ pub unsafe extern "C" fn dc_accounts_add_closed_account(accounts: *mut dc_accoun
         match accounts.add_closed_account().await {
             Ok(id) => id,
             Err(err) => {
-                accounts.emit_event(EventType::Error(format!(
-                    "Failed to add account: {:#}",
-                    err
-                )));
+                accounts.emit_event(EventType::Error(format!("Failed to add account: {err:#}")));
                 0
             }
         }
@@ -4455,8 +4628,7 @@ pub unsafe extern "C" fn dc_accounts_remove_account(
             Ok(()) => 1,
             Err(err) => {
                 accounts.emit_event(EventType::Error(format!(
-                    "Failed to remove account: {:#}",
-                    err
+                    "Failed to remove account: {err:#}"
                 )));
                 0
             }
@@ -4486,8 +4658,7 @@ pub unsafe extern "C" fn dc_accounts_migrate_account(
             Ok(id) => id,
             Err(err) => {
                 accounts.emit_event(EventType::Error(format!(
-                    "Failed to migrate account: {:#}",
-                    err
+                    "Failed to migrate account: {err:#}"
                 )));
                 0
             }
@@ -4580,11 +4751,12 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
 
 #[cfg(feature = "jsonrpc")]
 mod jsonrpc {
-    use super::*;
     use deltachat_jsonrpc::api::CommandApi;
     use deltachat_jsonrpc::events::event_to_json_rpc_notification;
     use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession};
 
+    use super::*;
+
     pub struct dc_jsonrpc_instance_t {
         receiver: OutReceiver,
         handle: RpcSession<CommandApi>,
@@ -4600,33 +4772,22 @@ mod jsonrpc {
             return ptr::null_mut();
         }
 
-        let cmd_api =
-            deltachat_jsonrpc::api::CommandApi::from_arc((*account_manager).inner.clone());
+        let account_manager = &*account_manager;
+        let events = block_on(account_manager.read()).get_event_emitter();
+        let cmd_api = deltachat_jsonrpc::api::CommandApi::from_arc(account_manager.inner.clone());
 
         let (request_handle, receiver) = RpcClient::new();
-        let request_handle2 = request_handle.clone();
-        let handle = RpcSession::new(request_handle, cmd_api);
+        let handle = RpcSession::new(request_handle.clone(), cmd_api);
 
-        let events = block_on({
-            async {
-                let am = (*account_manager).inner.clone();
-                let ev = am.read().await.get_event_emitter();
-                drop(am);
-                ev
-            }
-        });
-
-        let event_thread = spawn({
-            async move {
-                while let Some(event) = events.recv().await {
-                    let event = event_to_json_rpc_notification(event);
-                    request_handle2
-                        .send_notification("event", Some(event))
-                        .await?;
-                }
-                let res: Result<(), anyhow::Error> = Ok(());
-                res
+        let event_thread = spawn(async move {
+            while let Some(event) = events.recv().await {
+                let event = event_to_json_rpc_notification(event);
+                request_handle
+                    .send_notification("event", Some(event))
+                    .await?;
             }
+            let res: Result<(), anyhow::Error> = Ok(());
+            res
         });
 
         let instance = dc_jsonrpc_instance_t {
@@ -4648,6 +4809,12 @@ mod jsonrpc {
         drop(Box::from_raw(jsonrpc_instance));
     }
 
+    fn spawn_handle_jsonrpc_request(handle: RpcSession<CommandApi>, request: String) {
+        spawn(async move {
+            handle.handle_incoming(&request).await;
+        });
+    }
+
     #[no_mangle]
     pub unsafe extern "C" fn dc_jsonrpc_request(
         jsonrpc_instance: *mut dc_jsonrpc_instance_t,
@@ -4658,12 +4825,9 @@ mod jsonrpc {
             return;
         }
 
-        let api = &*jsonrpc_instance;
-        let handle = &api.handle;
+        let handle = &(*jsonrpc_instance).handle;
         let request = to_string_lossy(request);
-        spawn(async move {
-            handle.handle_incoming(&request).await;
-        });
+        spawn_handle_jsonrpc_request(handle.clone(), request);
     }
 
     #[no_mangle]

deltachat-ffi/src/lot.rs

@@ -1,10 +1,12 @@
 //! # Legacy generic return values for C API.
 
+use std::borrow::Cow;
+
+use anyhow::Error;
+
 use crate::message::MessageState;
 use crate::qr::Qr;
 use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
 
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
@@ -12,6 +14,8 @@ use std::borrow::Cow;
 /// eg. by chatlist.get_summary() or dc_msg_get_summary().
 ///
 /// *Lot* is used in the meaning *heap* here.
+// The QR code grew too large.  So be it.
+#[allow(clippy::large_enum_variant)]
 #[derive(Debug)]
 pub enum Lot {
     Summary(Summary),
@@ -20,20 +24,15 @@ pub enum Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub enum Meaning {
+    #[default]
     None = 0,
     Text1Draft = 1,
     Text1Username = 2,
     Text1Self = 3,
 }
 
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
 impl Lot {
     pub fn get_text1(&self) -> Option<&str> {
         match self {
@@ -50,6 +49,7 @@ impl Lot {
                 Qr::FprMismatch { .. } => None,
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
+                Qr::Backup { .. } => None,
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
                 Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
@@ -101,6 +101,7 @@ impl Lot {
                 Qr::FprMismatch { .. } => LotState::QrFprMismatch,
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
+                Qr::Backup { .. } => LotState::QrBackup,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
                 Qr::Addr { .. } => LotState::QrAddr,
                 Qr::Url { .. } => LotState::QrUrl,
@@ -125,6 +126,7 @@ impl Lot {
                 Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
+                Qr::Backup { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
@@ -149,9 +151,9 @@ impl Lot {
 }
 
 #[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub enum LotState {
-    // Default
+    #[default]
     Undefined = 0,
 
     // Qr States
@@ -173,6 +175,8 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
+    QrBackup = 251,
+
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 
@@ -213,12 +217,6 @@ pub enum LotState {
     MsgOutMdnRcvd = 28,
 }
 
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
 impl From<MessageState> for LotState {
     fn from(s: MessageState) -> Self {
         use MessageState::*;

deltachat-ffi/src/string.rs

@@ -287,9 +287,10 @@ fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use libc::{free, strcmp};
 
+    use super::*;
+
     #[test]
     fn test_os_str_to_c_string_cwd() {
         let some_dir = std::env::current_dir().unwrap();

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.105.0"
+version = "1.111.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -19,22 +19,24 @@ serde = { version = "1.0", features = ["derive"] }
 tempfile = "3.3.0"
 log = "0.4"
 async-channel = { version = "1.8.0" }
-futures = { version = "0.3.25" }
+futures = { version = "0.3.26" }
 serde_json = "1.0.91"
-yerpc = { version = "^0.3.1", features = ["anyhow_expose"] }
+yerpc = { version = "0.4.3", features = ["anyhow_expose"] }
 typescript-type-def = { version = "0.5.5", features = ["json_value"] }
-tokio = { version = "1.23.1" }
+tokio = { version = "1.25.0" }
 sanitize-filename = "0.4"
 walkdir = "2.3.2"
+base64 = "0.21"
 
 # optional dependencies
-axum = { version = "0.6.1", optional = true, features = ["ws"] }
+axum = { version = "0.6.11", optional = true, features = ["ws"] }
 env_logger = { version = "0.10.0", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.23.1", features = ["full", "rt-multi-thread"] }
+tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] }
 
 
 [features]
-default = []
-webserver = ["env_logger", "axum", "tokio/full", "yerpc/support-axum"]
+default = ["vendored"]
+webserver = ["dep:env_logger", "dep:axum", "tokio/full", "yerpc/support-axum"]
+vendored = ["deltachat/vendored"]

deltachat-jsonrpc/README.md

@@ -35,7 +35,7 @@ The server can be configured with environment variables:
 |`DC_PORT`|`20808`|port to listen on|
 |`DC_ACCOUNTS_PATH`|`./accounts`|path to storage directory|
 
-If you are targetting other architectures (like KaiOS or Android), the webserver binary can be cross-compiled easily with [rust-cross](https://github.com/cross-rs/cross):
+If you are targeting other architectures (like KaiOS or Android), the webserver binary can be cross-compiled easily with [rust-cross](https://github.com/cross-rs/cross):
 
 ```sh
 cross build --features=webserver --target armv7-linux-androideabi --release

deltachat-jsonrpc/src/api/mod.rs

@@ -1,8 +1,15 @@
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::{collections::HashMap, str::FromStr};
+
 use anyhow::{anyhow, bail, ensure, Context, Result};
+pub use deltachat::accounts::Accounts;
+use deltachat::qr::Qr;
 use deltachat::{
     chat::{
-        self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, marknoticed_chat,
-        remove_contact_from_chat, Chat, ChatId, ChatItem, ProtectionStatus,
+        self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, get_chat_msgs_ex,
+        marknoticed_chat, remove_contact_from_chat, Chat, ChatId, ChatItem, MessageListOptions,
+        ProtectionStatus,
     },
     chatlist::Chatlist,
     config::Config,
@@ -16,36 +23,32 @@ use deltachat::{
     },
     provider::get_provider_info,
     qr,
-    qr_code_generator::get_securejoin_qr_svg,
+    qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg},
     reaction::send_reaction,
     securejoin,
     stock_str::StockMessage,
     webxdc::StatusUpdateSerial,
 };
 use sanitize_filename::is_sanitized;
-use std::collections::BTreeMap;
-use std::sync::Arc;
-use std::{collections::HashMap, str::FromStr};
-use tokio::{fs, sync::RwLock};
+use tokio::fs;
+use tokio::sync::{watch, Mutex, RwLock};
 use walkdir::WalkDir;
 use yerpc::rpc;
 
-pub use deltachat::accounts::Accounts;
-
 pub mod events;
 pub mod types;
 
-use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult};
-use crate::api::types::qr::QrObject;
-
+use num_traits::FromPrimitive;
 use types::account::Account;
 use types::chat::FullChat;
 use types::chat_list::ChatListEntry;
 use types::contact::ContactObject;
+use types::message::MessageData;
 use types::message::MessageObject;
 use types::provider_info::ProviderInfo;
 use types::webxdc::WebxdcMessageInfo;
 
+use self::types::message::MessageLoadResult;
 use self::types::{
     chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
     location::JsonrpcLocation,
@@ -53,24 +56,48 @@ use self::types::{
         JSONRPCMessageListItem, MessageNotificationInfo, MessageSearchResult, MessageViewtype,
     },
 };
+use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult};
+use crate::api::types::qr::QrObject;
 
-use num_traits::FromPrimitive;
+#[derive(Debug)]
+struct AccountState {
+    /// The Qr code for current [`CommandApi::provide_backup`] call.
+    ///
+    /// If there currently is a call to [`CommandApi::provide_backup`] this will be
+    /// `Pending` or `Ready`, otherwise `NoProvider`.
+    backup_provider_qr: watch::Sender<ProviderQr>,
+}
+
+impl Default for AccountState {
+    fn default() -> Self {
+        let (tx, _rx) = watch::channel(ProviderQr::NoProvider);
+        Self {
+            backup_provider_qr: tx,
+        }
+    }
+}
 
 #[derive(Clone, Debug)]
 pub struct CommandApi {
     pub(crate) accounts: Arc<RwLock<Accounts>>,
+
+    states: Arc<Mutex<BTreeMap<u32, AccountState>>>,
 }
 
 impl CommandApi {
     pub fn new(accounts: Accounts) -> Self {
         CommandApi {
             accounts: Arc::new(RwLock::new(accounts)),
+            states: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
 
     #[allow(dead_code)]
     pub fn from_arc(accounts: Arc<RwLock<Accounts>>) -> Self {
-        CommandApi { accounts }
+        CommandApi {
+            accounts,
+            states: Arc::new(Mutex::new(BTreeMap::new())),
+        }
     }
 
     async fn get_context(&self, id: u32) -> Result<deltachat::context::Context> {
@@ -82,10 +109,47 @@ impl CommandApi {
             .ok_or_else(|| anyhow!("account with id {} not found", id))?;
         Ok(sc)
     }
+
+    async fn with_state<F, T>(&self, id: u32, with_state: F) -> T
+    where
+        F: FnOnce(&AccountState) -> T,
+    {
+        let mut states = self.states.lock().await;
+        let state = states.entry(id).or_insert_with(Default::default);
+        with_state(state)
+    }
+
+    async fn inner_get_backup_qr(&self, account_id: u32) -> Result<Qr> {
+        let mut receiver = self
+            .with_state(account_id, |state| state.backup_provider_qr.subscribe())
+            .await;
+
+        let val: ProviderQr = receiver.borrow_and_update().clone();
+        match val {
+            ProviderQr::NoProvider => bail!("No backup being provided"),
+            ProviderQr::Pending => loop {
+                if receiver.changed().await.is_err() {
+                    bail!("No backup being provided (account state dropped)");
+                }
+                let val: ProviderQr = receiver.borrow().clone();
+                match val {
+                    ProviderQr::NoProvider => bail!("No backup being provided"),
+                    ProviderQr::Pending => continue,
+                    ProviderQr::Ready(qr) => break Ok(qr),
+                };
+            },
+            ProviderQr::Ready(qr) => Ok(qr),
+        }
+    }
 }
 
 #[rpc(all_positional, ts_outdir = "typescript/generated")]
 impl CommandApi {
+    /// Test function.
+    async fn sleep(&self, delay: f64) {
+        tokio::time::sleep(std::time::Duration::from_secs_f64(delay)).await
+    }
+
     // ---------------------------------------------
     //  Misc top level functions
     // ---------------------------------------------
@@ -109,7 +173,13 @@ impl CommandApi {
     }
 
     async fn remove_account(&self, account_id: u32) -> Result<()> {
-        self.accounts.write().await.remove_account(account_id).await
+        self.accounts
+            .write()
+            .await
+            .remove_account(account_id)
+            .await?;
+        self.states.lock().await.remove(&account_id);
+        Ok(())
     }
 
     async fn get_all_account_ids(&self) -> Vec<u32> {
@@ -136,7 +206,7 @@ impl CommandApi {
             if let Some(ctx) = context_option {
                 accounts.push(Account::from_context(&ctx, id).await?)
             } else {
-                println!("account with id {} doesn't exist anymore", id);
+                println!("account with id {id} doesn't exist anymore");
             }
         }
         Ok(accounts)
@@ -244,7 +314,7 @@ impl CommandApi {
         for (key, value) in config.into_iter() {
             set_config(&ctx, &key, value.as_deref())
                 .await
-                .with_context(|| format!("Can't set {} to {:?}", key, value))?;
+                .with_context(|| format!("Can't set {key} to {value:?}"))?;
         }
         Ok(())
     }
@@ -461,7 +531,7 @@ impl CommandApi {
                     Ok(res) => res,
                     Err(err) => ChatListItemFetchResult::Error {
                         id: entry.0,
-                        error: format!("{:?}", err),
+                        error: format!("{err:#}"),
                     },
                 },
             );
@@ -805,7 +875,7 @@ impl CommandApi {
         let ctx = self.get_context(account_id).await?;
 
         // TODO: implement this in core with an SQL query, that will be way faster
-        let messages = get_chat_msgs(&ctx, ChatId::new(chat_id), 0).await?;
+        let messages = get_chat_msgs(&ctx, ChatId::new(chat_id)).await?;
         let mut first_unread_message_id = None;
         for item in messages.into_iter().rev() {
             if let ChatItem::Message { msg_id } = item {
@@ -880,9 +950,23 @@ impl CommandApi {
         markseen_msgs(&ctx, msg_ids.into_iter().map(MsgId::new).collect()).await
     }
 
-    async fn get_message_ids(&self, account_id: u32, chat_id: u32, flags: u32) -> Result<Vec<u32>> {
+    async fn get_message_ids(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+        info_only: bool,
+        add_daymarker: bool,
+    ) -> Result<Vec<u32>> {
         let ctx = self.get_context(account_id).await?;
-        let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?;
+        let msg = get_chat_msgs_ex(
+            &ctx,
+            ChatId::new(chat_id),
+            MessageListOptions {
+                info_only,
+                add_daymarker,
+            },
+        )
+        .await?;
         Ok(msg
             .iter()
             .map(|chat_item| -> u32 {
@@ -898,10 +982,19 @@ impl CommandApi {
         &self,
         account_id: u32,
         chat_id: u32,
-        flags: u32,
+        info_only: bool,
+        add_daymarker: bool,
     ) -> Result<Vec<JSONRPCMessageListItem>> {
         let ctx = self.get_context(account_id).await?;
-        let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?;
+        let msg = get_chat_msgs_ex(
+            &ctx,
+            ChatId::new(chat_id),
+            MessageListOptions {
+                info_only,
+                add_daymarker,
+            },
+        )
+        .await?;
         Ok(msg
             .iter()
             .map(|chat_item| (*chat_item).into())
@@ -918,17 +1011,27 @@ impl CommandApi {
         MsgId::new(message_id).get_html(&ctx).await
     }
 
+    /// get multiple messages in one call,
+    /// if loading one message fails the error is stored in the result object in it's place.
+    ///
+    /// this is the batch variant of [get_message]
     async fn get_messages(
         &self,
         account_id: u32,
         message_ids: Vec<u32>,
-    ) -> Result<HashMap<u32, MessageObject>> {
+    ) -> Result<HashMap<u32, MessageLoadResult>> {
         let ctx = self.get_context(account_id).await?;
-        let mut messages: HashMap<u32, MessageObject> = HashMap::new();
+        let mut messages: HashMap<u32, MessageLoadResult> = HashMap::new();
         for message_id in message_ids {
+            let message_result = MessageObject::from_message_id(&ctx, message_id).await;
             messages.insert(
                 message_id,
-                MessageObject::from_message_id(&ctx, message_id).await?,
+                match message_result {
+                    Ok(message) => MessageLoadResult::Message(message),
+                    Err(error) => MessageLoadResult::LoadingError {
+                        error: format!("{error:#}"),
+                    },
+                },
             );
         }
         Ok(messages)
@@ -1286,16 +1389,13 @@ impl CommandApi {
         passphrase: Option<String>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        ctx.stop_io().await;
-        let result = imex::imex(
+        imex::imex(
             &ctx,
             imex::ImexMode::ExportBackup,
             destination.as_ref(),
             passphrase,
         )
-        .await;
-        ctx.start_io().await;
-        result
+        .await
     }
 
     async fn import_backup(
@@ -1314,6 +1414,75 @@ impl CommandApi {
         .await
     }
 
+    /// Offers a backup for remote devices to retrieve.
+    ///
+    /// Can be cancelled by stopping the ongoing process.  Success or failure can be tracked
+    /// via the `ImexProgress` event which should either reach `1000` for success or `0` for
+    /// failure.
+    ///
+    /// This **stops IO** while it is running.
+    ///
+    /// Returns once a remote device has retrieved the backup, or is cancelled.
+    async fn provide_backup(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        self.with_state(account_id, |state| {
+            state.backup_provider_qr.send_replace(ProviderQr::Pending);
+        })
+        .await;
+
+        let provider = imex::BackupProvider::prepare(&ctx).await?;
+        self.with_state(account_id, |state| {
+            state
+                .backup_provider_qr
+                .send_replace(ProviderQr::Ready(provider.qr()));
+        })
+        .await;
+
+        provider.await
+    }
+
+    /// Returns the text of the QR code for the running [`CommandApi::provide_backup`].
+    ///
+    /// This QR code text can be used in [`CommandApi::get_backup`] on a second device to
+    /// retrieve the backup and setup this second device.
+    ///
+    /// This call will fail if there is currently no concurrent call to
+    /// [`CommandApi::provide_backup`].  This call may block if the QR code is not yet
+    /// ready.
+    async fn get_backup_qr(&self, account_id: u32) -> Result<String> {
+        let qr = self.inner_get_backup_qr(account_id).await?;
+        qr::format_backup(&qr)
+    }
+
+    /// Returns the rendered QR code for the running [`CommandApi::provide_backup`].
+    ///
+    /// This QR code can be used in [`CommandApi::get_backup`] on a second device to
+    /// retrieve the backup and setup this second device.
+    ///
+    /// This call will fail if there is currently no concurrent call to
+    /// [`CommandApi::provide_backup`].  This call may block if the QR code is not yet
+    /// ready.
+    ///
+    /// Returns the QR code rendered as an SVG image.
+    async fn get_backup_qr_svg(&self, account_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let qr = self.inner_get_backup_qr(account_id).await?;
+        generate_backup_qr(&ctx, &qr).await
+    }
+
+    /// Gets a backup from a remote provider.
+    ///
+    /// This retrieves the backup from a remote device over the network and imports it into
+    /// the current device.
+    ///
+    /// Can be cancelled by stopping the ongoing process.
+    async fn get_backup(&self, account_id: u32, qr_text: String) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let qr = qr::check_qr(&ctx, &qr_text).await?;
+        imex::get_backup(&ctx, qr).await?;
+        Ok(())
+    }
+
     // ---------------------------------------------
     //                connectivity
     // ---------------------------------------------
@@ -1424,6 +1593,23 @@ impl CommandApi {
         WebxdcMessageInfo::get_for_message(&ctx, MsgId::new(instance_msg_id)).await
     }
 
+    /// Get blob encoded as base64 from a webxdc message
+    ///
+    /// path is the path of the file within webxdc archive
+    async fn get_webxdc_blob(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+        path: String,
+    ) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let message = Message::load_from_db(&ctx, MsgId::new(instance_msg_id)).await?;
+        let blob = message.get_webxdc_blob(&ctx, &path).await?;
+
+        use base64::{engine::general_purpose, Engine as _};
+        Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
+    }
+
     /// Forward messages to another chat.
     ///
     /// All types of messages can be forwarded,
@@ -1473,6 +1659,48 @@ impl CommandApi {
         Ok(message_id.to_u32())
     }
 
+    async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        let mut message = Message::new(if let Some(viewtype) = data.viewtype {
+            viewtype.into()
+        } else if data.file.is_some() {
+            Viewtype::File
+        } else {
+            Viewtype::Text
+        });
+        if data.text.is_some() {
+            message.set_text(data.text);
+        }
+        if data.html.is_some() {
+            message.set_html(data.html);
+        }
+        if data.override_sender_name.is_some() {
+            message.set_override_sender_name(data.override_sender_name);
+        }
+        if let Some(file) = data.file {
+            message.set_file(file, None);
+        }
+        if let Some((latitude, longitude)) = data.location {
+            message.set_location(latitude, longitude);
+        }
+        if let Some(id) = data.quoted_message_id {
+            message
+                .set_quote(
+                    &ctx,
+                    Some(
+                        &Message::load_from_db(&ctx, MsgId::new(id))
+                            .await
+                            .context("message to quote could not be loaded")?,
+                    ),
+                )
+                .await?;
+        }
+        let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
+            .await?
+            .to_u32();
+        Ok(msg_id)
+    }
+
     // ---------------------------------------------
     //           functions for the composer
     //    the composer is the message input field
@@ -1716,7 +1944,7 @@ async fn set_config(
         ctx.set_ui_config(key, value).await?;
     } else {
         ctx.set_config(
-            Config::from_str(key).with_context(|| format!("unknown key {:?}", key))?,
+            Config::from_str(key).with_context(|| format!("unknown key {key:?}"))?,
             value,
         )
         .await?;
@@ -1738,7 +1966,19 @@ async fn get_config(
     if key.starts_with("ui.") {
         ctx.get_ui_config(key).await
     } else {
-        ctx.get_config(Config::from_str(key).with_context(|| format!("unknown key {:?}", key))?)
+        ctx.get_config(Config::from_str(key).with_context(|| format!("unknown key {key:?}"))?)
             .await
     }
 }
+
+/// Whether a QR code for a BackupProvider is currently available.
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone, Debug)]
+enum ProviderQr {
+    /// There is no provider, asking for a QR is an error.
+    NoProvider,
+    /// There is a provider, the QR code is pending.
+    Pending,
+    /// There is a provider and QR code.
+    Ready(Qr),
+}

deltachat-jsonrpc/src/api/types/contact.rs

@@ -20,6 +20,10 @@ pub struct ContactObject {
     name_and_addr: String,
     is_blocked: bool,
     is_verified: bool,
+    /// the address that verified this contact
+    verifier_addr: Option<String>,
+    /// the id of the contact that verified this contact
+    verifier_id: Option<u32>,
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
@@ -36,6 +40,18 @@ impl ContactObject {
         };
         let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
 
+        let (verifier_addr, verifier_id) = if is_verified {
+            (
+                contact.get_verifier_addr(context).await?,
+                contact
+                    .get_verifier_id(context)
+                    .await?
+                    .map(|contact_id| contact_id.to_u32()),
+            )
+        } else {
+            (None, None)
+        };
+
         Ok(ContactObject {
             address: contact.get_addr().to_owned(),
             color: color_int_to_hex_string(contact.get_color()),
@@ -48,6 +64,8 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
+            verifier_addr,
+            verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),
         })

deltachat-jsonrpc/src/api/types/message.rs

@@ -19,6 +19,13 @@ use super::contact::ContactObject;
 use super::reactions::JSONRPCReactions;
 use super::webxdc::WebxdcMessageInfo;
 
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase", tag = "variant")]
+pub enum MessageLoadResult {
+    Message(MessageObject),
+    LoadingError { error: String },
+}
+
 #[derive(Serialize, TypeDef)]
 #[serde(rename = "Message", rename_all = "camelCase")]
 pub struct MessageObject {
@@ -48,6 +55,10 @@ pub struct MessageObject {
     is_setupmessage: bool,
     is_info: bool,
     is_forwarded: bool,
+
+    /// True if the message was sent by a bot.
+    is_bot: bool,
+
     /// when is_info is true this describes what type of system message it is
     system_message_type: SystemMessageType,
 
@@ -182,6 +193,7 @@ impl MessageObject {
             is_setupmessage: message.is_setupmessage(),
             is_info: message.is_info(),
             is_forwarded: message.is_forwarded(),
+            is_bot: message.is_bot(),
             system_message_type: message.get_info_type().into(),
 
             duration: message.get_duration(),
@@ -490,3 +502,15 @@ impl From<ChatItem> for JSONRPCMessageListItem {
         }
     }
 }
+
+#[derive(Deserialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageData {
+    pub text: Option<String>,
+    pub html: Option<String>,
+    pub viewtype: Option<MessageViewtype>,
+    pub file: Option<String>,
+    pub location: Option<(f64, f64)>,
+    pub override_sender_name: Option<String>,
+    pub quoted_message_id: Option<u32>,
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -10,7 +10,7 @@ pub mod reactions;
 pub mod webxdc;
 
 pub fn color_int_to_hex_string(color: u32) -> String {
-    format!("{:#08x}", color).replace("0x", "#")
+    format!("{color:#08x}").replace("0x", "#")
 }
 
 fn maybe_empty_string_to_option(string: String) -> Option<String> {

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -8,7 +8,7 @@ use typescript_type_def::TypeDef;
 pub struct ProviderInfo {
     pub before_login_hint: String,
     pub overview_page: String,
-    pub status: u32, // in reality this is an enum, but for simlicity and because it gets converted into a number anyway, we use an u32 here.
+    pub status: u32, // in reality this is an enum, but for simplicity and because it gets converted into a number anyway, we use an u32 here.
 }
 
 impl ProviderInfo {

deltachat-jsonrpc/src/api/types/qr.rs

@@ -32,6 +32,9 @@ pub enum QrObject {
     Account {
         domain: String,
     },
+    Backup {
+        ticket: String,
+    },
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
@@ -126,6 +129,9 @@ impl From<Qr> for QrObject {
             }
             Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint },
             Qr::Account { domain } => QrObject::Account { domain },
+            Qr::Backup { ticket } => QrObject::Backup {
+                ticket: ticket.to_string(),
+            },
             Qr::WebrtcInstance {
                 domain,
                 instance_pattern,

deltachat-jsonrpc/src/lib.rs

@@ -4,12 +4,13 @@ pub use yerpc;
 
 #[cfg(test)]
 mod tests {
-    use super::api::{Accounts, CommandApi};
     use async_channel::unbounded;
     use futures::StreamExt;
     use tempfile::TempDir;
     use yerpc::{RpcClient, RpcSession};
 
+    use super::api::{Accounts, CommandApi};
+
     #[tokio::test(flavor = "multi_thread")]
     async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
         let tmp_dir = TempDir::new().unwrap().path().into();
@@ -36,7 +37,7 @@ mod tests {
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
             let result = receiver.next().await;
-            println!("{:?}", result);
+            println!("{result:?}");
             assert_eq!(result, Some(response.to_owned()));
         }
         {
@@ -44,7 +45,7 @@ mod tests {
             let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
             session.handle_incoming(request).await;
             let result = receiver.next().await;
-            println!("{:?}", result);
+            println!("{result:?}");
             assert_eq!(result, Some(response.to_owned()));
         }
 

deltachat-jsonrpc/src/webserver.rs

@@ -1,6 +1,7 @@
-use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router};
 use std::net::SocketAddr;
 use std::path::PathBuf;
+
+use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router};
 use yerpc::axum::handle_ws_rpc;
 use yerpc::{RpcClient, RpcSession};
 

deltachat-jsonrpc/typescript/.npmignore

@@ -4,3 +4,9 @@ docs
 coverage
 yarn*
 package-lock.json
+.prettierignore
+example.html
+report_api_coverage.mjs
+scripts
+dist/example
+dist/test

deltachat-jsonrpc/typescript/example/example.ts

@@ -68,22 +68,22 @@ async function run() {
       null
     );
     for (const [chatId, _messageId] of chats) {
-      const chat = await client.rpc.getFullChatById(
-        selectedAccount,
-        chatId
-      );
+      const chat = await client.rpc.getFullChatById(selectedAccount, chatId);
       write($main, `<h3>${chat.name}</h3>`);
       const messageIds = await client.rpc.getMessageIds(
         selectedAccount,
         chatId,
-        0
+        false,
+        false
       );
       const messages = await client.rpc.getMessages(
         selectedAccount,
         messageIds
       );
       for (const [_messageId, message] of Object.entries(messages)) {
-        write($main, `<p>${message.text}</p>`);
+        if (message.variant === "message")
+          write($main, `<p>${message.text}</p>`);
+        else write($main, `<p>loading error: ${message.error}</p>`);
       }
     }
   }

deltachat-jsonrpc/typescript/example/node-add-account.js

@@ -3,24 +3,27 @@ import { DeltaChat } from "../dist/deltachat.js";
 run().catch(console.error);
 
 async function run() {
-  const delta = new DeltaChat('ws://localhost:20808/ws');
+  const delta = new DeltaChat("ws://localhost:20808/ws");
   delta.on("event", (event) => {
     console.log("event", event.data);
   });
 
-  const email = process.argv[2]
-  const password = process.argv[3]
-  if (!email || !password) throw new Error('USAGE: node node-add-account.js <EMAILADDRESS> <PASSWORD>')
-  console.log(`creating acccount for ${email}`)
-  const id = await delta.rpc.addAccount()
-  console.log(`created account id ${id}`)
+  const email = process.argv[2];
+  const password = process.argv[3];
+  if (!email || !password)
+    throw new Error(
+      "USAGE: node node-add-account.js <EMAILADDRESS> <PASSWORD>"
+    );
+  console.log(`creating account for ${email}`);
+  const id = await delta.rpc.addAccount();
+  console.log(`created account id ${id}`);
   await delta.rpc.setConfig(id, "addr", email);
   await delta.rpc.setConfig(id, "mail_pw", password);
-  console.log('configuration updated')
-  await delta.rpc.configure(id)
-  console.log('account configured!')
+  console.log("configuration updated");
+  await delta.rpc.configure(id);
+  console.log("account configured!");
 
   const accounts = await delta.rpc.getAllAccounts();
   console.log("accounts", accounts);
-  console.log("waiting for events...")
+  console.log("waiting for events...");
 }

deltachat-jsonrpc/typescript/example/node-demo.js

@@ -10,5 +10,5 @@ async function run() {
 
   const accounts = await delta.rpc.getAllAccounts();
   console.log("accounts", accounts);
-  console.log("waiting for events...")
+  console.log("waiting for events...");
 }

deltachat-jsonrpc/typescript/package.json

@@ -3,7 +3,7 @@
   "dependencies": {
     "@deltachat/tiny-emitter": "3.0.0",
     "isomorphic-ws": "^4.0.1",
-    "yerpc": "^0.3.3"
+    "yerpc": "^0.4.3"
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
@@ -14,7 +14,7 @@
     "c8": "^7.10.0",
     "chai": "^4.3.4",
     "chai-as-promised": "^7.1.1",
-    "esbuild": "^0.14.11",
+    "esbuild": "^0.17.9",
     "http-server": "^14.1.1",
     "mocha": "^9.1.1",
     "node-fetch": "^2.6.1",
@@ -24,12 +24,19 @@
     "typescript": "^4.5.5",
     "ws": "^8.5.0"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/deltachat.js",
+      "require": "./dist/deltachat.cjs"
+    }
+  },
   "license": "MPL-2.0",
   "main": "dist/deltachat.js",
   "name": "@deltachat/jsonrpc-client",
   "scripts": {
-    "build": "run-s generate-bindings extract-constants build:tsc build:bundle",
+    "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
     "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
+    "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs",
     "build:tsc": "tsc",
     "docs": "typedoc --out docs deltachat.ts",
     "example": "run-s build example:build example:start",
@@ -38,8 +45,8 @@
     "example:start": "http-server .",
     "extract-constants": "node ./scripts/generate-constants.js",
     "generate-bindings": "cargo test",
-    "prettier:check": "prettier --check **.ts",
-    "prettier:fix": "prettier --write **.ts",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write .",
     "test": "run-s test:prepare test:run-coverage test:report-coverage",
     "test:prepare": "cargo build --package deltachat-rpc-server --bin deltachat-rpc-server",
     "test:report-coverage": "node report_api_coverage.mjs",
@@ -48,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.105.0"
-}
+  "version": "1.111.0"
+}

deltachat-jsonrpc/typescript/report_api_coverage.mjs

@@ -1,5 +1,5 @@
 import { readFileSync } from "fs";
-// only checks for the coverge of the api functions in bindings.ts for now
+// only checks for the coverage of the api functions in bindings.ts for now
 const generatedFile = "typescript/generated/client.ts";
 const json = JSON.parse(readFileSync("./coverage/coverage-final.json"));
 const jsonCoverage =

deltachat-jsonrpc/typescript/src/client.ts

@@ -43,7 +43,7 @@ export class BaseDeltaChat<
       const method = request.method;
       if (method === "event") {
         const event = request.params! as DCWireEvent<Event>;
-	//@ts-ignore
+        //@ts-ignore
         this.emit(event.event.type, event.contextId, event.event as any);
         this.emit("ALL", event.contextId, event.event as any);
 
@@ -120,7 +120,7 @@ export class StdioTransport extends BaseTransport {
     });
   }
 
-  _send(message: RPC.Message): void {
+  _send(message: any): void {
     const serialized = JSON.stringify(message);
     this.input.write(serialized + "\n");
   }

deltachat-jsonrpc/typescript/test/basic.ts

@@ -4,10 +4,7 @@ import chaiAsPromised from "chai-as-promised";
 chai.use(chaiAsPromised);
 import { StdioDeltaChat as DeltaChat } from "../deltachat.js";
 
-import {
-  RpcServerHandle,
-  startServer,
-} from "./test_base.js";
+import { RpcServerHandle, startServer } from "./test_base.js";
 
 describe("basic tests", () => {
   let serverHandle: RpcServerHandle;
@@ -15,9 +12,9 @@ describe("basic tests", () => {
 
   before(async () => {
     serverHandle = await startServer();
-    dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout)
+    dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout);
     // dc.on("ALL", (event) => {
-      //console.log("event", event);
+    //console.log("event", event);
     // });
   });
 
@@ -56,7 +53,7 @@ describe("basic tests", () => {
     ]);
   });
 
-  describe("account managment", () => {
+  describe("account management", () => {
     it("should create account", async () => {
       const res = await dc.rpc.addAccount();
       assert((await dc.rpc.getAllAccountIds()).length === 1);
@@ -76,7 +73,7 @@ describe("basic tests", () => {
     });
   });
 
-  describe("contact managment", function () {
+  describe("contact management", function () {
     let accountId: number;
     before(async () => {
       accountId = await dc.rpc.addAccount();
@@ -106,38 +103,44 @@ describe("basic tests", () => {
       accountId = await dc.rpc.addAccount();
     });
 
-    it("set and retrive", async function () {
+    it("set and retrieve", async function () {
       await dc.rpc.setConfig(accountId, "addr", "valid@email");
       assert((await dc.rpc.getConfig(accountId, "addr")) == "valid@email");
     });
     it("set invalid key should throw", async function () {
-      await expect(dc.rpc.setConfig(accountId, "invalid_key", "some value")).to.be
-        .eventually.rejected;
+      await expect(dc.rpc.setConfig(accountId, "invalid_key", "some value")).to
+        .be.eventually.rejected;
     });
     it("get invalid key should throw", async function () {
       await expect(dc.rpc.getConfig(accountId, "invalid_key")).to.be.eventually
         .rejected;
     });
-    it("set and retrive ui.*", async function () {
+    it("set and retrieve ui.*", async function () {
       await dc.rpc.setConfig(accountId, "ui.chat_bg", "color:red");
       assert((await dc.rpc.getConfig(accountId, "ui.chat_bg")) == "color:red");
     });
-    it("set and retrive (batch)", async function () {
+    it("set and retrieve (batch)", async function () {
       const config = { addr: "valid@email", mail_pw: "1234" };
       await dc.rpc.batchSetConfig(accountId, config);
-      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
       expect(retrieved).to.deep.equal(config);
     });
-    it("set and retrive ui.* (batch)", async function () {
+    it("set and retrieve ui.* (batch)", async function () {
       const config = {
         "ui.chat_bg": "color:green",
         "ui.enter_key_sends": "true",
       };
       await dc.rpc.batchSetConfig(accountId, config);
-      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
       expect(retrieved).to.deep.equal(config);
     });
-    it("set and retrive mixed(ui and core) (batch)", async function () {
+    it("set and retrieve mixed(ui and core) (batch)", async function () {
       const config = {
         "ui.chat_bg": "color:yellow",
         "ui.enter_key_sends": "false",
@@ -145,7 +148,10 @@ describe("basic tests", () => {
         mail_pw: "123456",
       };
       await dc.rpc.batchSetConfig(accountId, config);
-      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
       expect(retrieved).to.deep.equal(config);
     });
   });

deltachat-jsonrpc/typescript/test/online.ts

@@ -22,7 +22,7 @@ describe("online tests", function () {
         process.exit(1);
       }
       console.log(
-        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip intergration tests"
+        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests"
       );
       this.skip();
     }
@@ -36,7 +36,7 @@ describe("online tests", function () {
     account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
     if (!account1 || !account1.email || !account1.password) {
       console.log(
-        "We didn't got back an account from the api, skip intergration tests"
+        "We didn't got back an account from the api, skip integration tests"
       );
       this.skip();
     }
@@ -44,7 +44,7 @@ describe("online tests", function () {
     account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
     if (!account2 || !account2.email || !account2.password) {
       console.log(
-        "We didn't got back an account2 from the api, skip intergration tests"
+        "We didn't got back an account2 from the api, skip integration tests"
       );
       this.skip();
     }
@@ -74,7 +74,7 @@ describe("online tests", function () {
     accountsConfigured = true;
   });
 
-  it("send and recieve text message", async function () {
+  it("send and receive text message", async function () {
     if (!accountsConfigured) {
       this.skip();
     }
@@ -97,7 +97,8 @@ describe("online tests", function () {
     const messageList = await dc.rpc.getMessageIds(
       accountId2,
       chatIdOnAccountB,
-      0
+      false,
+      false
     );
 
     expect(messageList).have.length(1);
@@ -105,7 +106,7 @@ describe("online tests", function () {
     expect(message.text).equal("Hello");
   });
 
-  it("send and recieve text message roundtrip, encrypted on answer onwards", async function () {
+  it("send and receive text message roundtrip, encrypted on answer onwards", async function () {
     if (!accountsConfigured) {
       this.skip();
     }
@@ -133,7 +134,8 @@ describe("online tests", function () {
     const messageList = await dc.rpc.getMessageIds(
       accountId2,
       chatIdOnAccountB,
-      0
+      false,
+      false
     );
     const message = await dc.rpc.getMessage(
       accountId2,
@@ -150,7 +152,7 @@ describe("online tests", function () {
     await eventPromise2;
 
     const messageId = (
-      await dc.rpc.getMessageIds(accountId1, chatId, 0)
+      await dc.rpc.getMessageIds(accountId1, chatId, false, false)
     ).reverse()[0];
     const message2 = await dc.rpc.getMessage(accountId1, messageId);
     expect(message2.text).equal("super secret message");

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -59,13 +59,13 @@ export async function startServer(): Promise<RpcServerHandle> {
 
 export async function createTempUser(url: string) {
   const response = await fetch(url, {
-    method: "POST", 
+    method: "POST",
     headers: {
       "cache-control": "no-cache",
     },
   });
-  if (!response.ok) throw new Error('Received invalid response')
-  return response.json(); 
+  if (!response.ok) throw new Error("Received invalid response");
+  return response.json();
 }
 
 function getTargetDir(): Promise<string> {
@@ -89,4 +89,3 @@ function getTargetDir(): Promise<string> {
     );
   });
 }
-

deltachat-ratelimit/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "ratelimit"
+version = "1.0.0"
+description = "Token bucket implementation"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]

deltachat-ratelimit/src/lib.rs

@@ -7,7 +7,7 @@
 use std::time::{Duration, SystemTime};
 
 #[derive(Debug)]
-pub(crate) struct Ratelimit {
+pub struct Ratelimit {
     /// Time of the last update.
     last_update: SystemTime,
 
@@ -25,7 +25,7 @@ impl Ratelimit {
     /// Returns a new rate limiter with the given constraints.
     ///
     /// Rate limiter will allow to send no more than `quota` messages within duration `window`.
-    pub(crate) fn new(window: Duration, quota: f64) -> Self {
+    pub fn new(window: Duration, quota: f64) -> Self {
         Self::new_at(window, quota, SystemTime::now())
     }
 
@@ -57,7 +57,7 @@ impl Ratelimit {
     /// Returns true if can send another message now.
     ///
     /// This method takes mutable reference
-    pub(crate) fn can_send(&self) -> bool {
+    pub fn can_send(&self) -> bool {
         self.can_send_at(SystemTime::now())
     }
 
@@ -71,7 +71,7 @@ impl Ratelimit {
     /// It is possible to send message even if over quota, e.g. if the message sending is initiated
     /// by the user and should not be rate limited. However, sending messages when over quota
     /// further postpones the time when it will be allowed to send low priority messages.
-    pub(crate) fn send(&mut self) {
+    pub fn send(&mut self) {
         self.send_at(SystemTime::now())
     }
 
@@ -87,7 +87,7 @@ impl Ratelimit {
     }
 
     /// Calculates the time until `can_send` will return `true`.
-    pub(crate) fn until_can_send(&self) -> Duration {
+    pub fn until_can_send(&self) -> Duration {
         self.until_can_send_at(SystemTime::now())
     }
 }

deltachat-repl/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "deltachat-repl"
+version = "1.111.0"
+license = "MPL-2.0"
+edition = "2021"
+
+[dependencies]
+ansi_term = "0.12.1"
+anyhow = "1"
+deltachat = { path = "..", features = ["internals"]}
+dirs = "4"
+log = "0.4.16"
+pretty_env_logger = "0.4"
+rusqlite = "0.28"
+rustyline = "11"
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+
+[features]
+default = ["vendored"]
+vendored = ["deltachat/vendored"]

deltachat-repl/src/cmdline.rs

@@ -29,13 +29,13 @@ use tokio::fs;
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
-/// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
+/// e.g. bitmask 7 triggers actions defined with bits 1, 2 and 4.
 async fn reset_tables(context: &Context, bits: i32) {
-    println!("Resetting tables ({})...", bits);
+    println!("Resetting tables ({bits})...");
     if 0 != bits & 1 {
         context
             .sql()
-            .execute("DELETE FROM jobs;", paramsv![])
+            .execute("DELETE FROM jobs;", ())
             .await
             .unwrap();
         println!("(1) Jobs reset.");
@@ -43,7 +43,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 2 {
         context
             .sql()
-            .execute("DELETE FROM acpeerstates;", paramsv![])
+            .execute("DELETE FROM acpeerstates;", ())
             .await
             .unwrap();
         println!("(2) Peerstates reset.");
@@ -51,7 +51,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 4 {
         context
             .sql()
-            .execute("DELETE FROM keypairs;", paramsv![])
+            .execute("DELETE FROM keypairs;", ())
             .await
             .unwrap();
         println!("(4) Private keypairs reset.");
@@ -59,36 +59,36 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 8 {
         context
             .sql()
-            .execute("DELETE FROM contacts WHERE id>9;", paramsv![])
+            .execute("DELETE FROM contacts WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats WHERE id>9;", paramsv![])
+            .execute("DELETE FROM chats WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats_contacts;", paramsv![])
+            .execute("DELETE FROM chats_contacts;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM msgs WHERE id>9;", paramsv![])
+            .execute("DELETE FROM msgs WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
             .execute(
                 "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-                paramsv![],
+                (),
             )
             .await
             .unwrap();
         context.sql().config_cache().write().await.clear();
         context
             .sql()
-            .execute("DELETE FROM leftgrps;", paramsv![])
+            .execute("DELETE FROM leftgrps;", ())
             .await
             .unwrap();
         println!("(8) Rest but server config reset.");
@@ -101,7 +101,7 @@ async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<
     let data = read_file(context, filename).await?;
 
     if let Err(err) = receive_imf(context, &data, false).await {
-        println!("receive_imf errored: {:?}", err);
+        println!("receive_imf errored: {err:?}");
     }
     Ok(())
 }
@@ -148,7 +148,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
-                    println!("Import: {}", path_plus_name);
+                    println!("Import: {path_plus_name}");
                     if poke_eml_file(context, path_plus_name).await.is_ok() {
                         read_cnt += 1
                     }
@@ -168,7 +168,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         .await
         .expect("invalid contact");
     let contact_name = if let Some(name) = msg.get_override_sender_name() {
-        format!("~{}", name)
+        format!("~{name}")
     } else {
         contact.get_display_name().to_string()
     };
@@ -223,7 +223,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
                     "[WEBXDC: {}, icon={}, document={}, summary={}, source_code_url={}]",
                     info.name, info.icon, info.document, info.summary, info.source_code_url
                 ),
-                Err(err) => format!("[get_webxdc_info() failed: {}]", err),
+                Err(err) => format!("[get_webxdc_info() failed: {err}]"),
             }
         } else {
             "".to_string()
@@ -336,6 +336,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  has-backup\n\
                  export-backup\n\
                  import-backup <backup-file>\n\
+                 send-backup\n\
+                 receive-backup <qr>\n\
                  export-keys\n\
                  import-keys\n\
                  export-setup\n\
@@ -435,10 +437,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ),
         },
         "initiate-key-transfer" => match initiate_key_transfer(&context).await {
-            Ok(setup_code) => println!(
-                "Setup code for the transferred setup message: {}",
-                setup_code,
-            ),
+            Ok(setup_code) => {
+                println!("Setup code for the transferred setup message: {setup_code}",)
+            }
             Err(err) => bail!("Failed to generate setup code: {}", err),
         },
         "get-setupcodebegin" => {
@@ -487,6 +488,17 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             )
             .await?;
         }
+        "send-backup" => {
+            let provider = BackupProvider::prepare(&context).await?;
+            let qr = provider.qr();
+            println!("QR code: {}", format_backup(&qr)?);
+            provider.await?;
+        }
+        "receive-backup" => {
+            ensure!(!arg1.is_empty(), "Argument <qr> is missing.");
+            let qr = check_qr(&context, arg1).await?;
+            deltachat::imex::get_backup(&context, qr).await?;
+        }
         "export-keys" => {
             let dir = dirs::home_dir().unwrap_or_default();
             imex(&context, ImexMode::ExportSelfKeys, dir.as_ref(), None).await?;
@@ -528,7 +540,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(arg1)?;
             let val = context.get_config(key).await;
-            println!("{}={:?}", key, val);
+            println!("{key}={val:?}");
         }
         "info" => {
             println!("{:#?}", context.get_info().await);
@@ -540,7 +552,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             match context.get_connectivity_html().await {
                 Ok(html) => {
                     fs::write(&file, html).await?;
-                    println!("Report written to: {:#?}", file);
+                    println!("Report written to: {file:#?}");
                 }
                 Err(err) => {
                     bail!("Failed to get connectivity html: {}", err);
@@ -613,7 +625,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "{}{}{} [{}]{}",
                         summary
                             .prefix
-                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
+                            .map_or_else(String::new, |prefix| format!("{prefix}: ")),
                         summary.text,
                         statestr,
                         &timestr,
@@ -631,8 +643,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             if location::is_sending_locations_to_chat(&context, None).await? {
                 println!("Location streaming enabled.");
             }
-            println!("{} chats", cnt);
-            println!("{:?} to create this list", time_needed);
+            println!("{cnt} chats");
+            println!("{time_needed:?} to create this list");
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -640,7 +652,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             if !arg1.is_empty() {
                 let id = ChatId::new(arg1.parse()?);
-                println!("Selecting chat {}", id);
+                println!("Selecting chat {id}");
                 sel_chat = Some(Chat::load_from_db(&context, id).await?);
                 *chat_id = id;
             }
@@ -649,8 +661,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let time_start = std::time::SystemTime::now();
-            let msglist =
-                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER).await?;
+            let msglist = chat::get_chat_msgs_ex(
+                &context,
+                sel_chat.get_id(),
+                chat::MessageListOptions {
+                    info_only: false,
+                    add_daymarker: true,
+                },
+            )
+            .await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             let msglist: Vec<MsgId> = msglist
@@ -686,7 +705,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 },
                 match sel_chat.get_profile_image(&context).await? {
                     Some(icon) => match icon.to_str() {
-                        Some(icon) => format!(" Icon: {}", icon),
+                        Some(icon) => format!(" Icon: {icon}"),
                         _ => " Icon: Err".to_string(),
                     },
                     _ => "".to_string(),
@@ -712,8 +731,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
 
             println!(
-                "{:?} to create this list, {:?} to mark all messages as noticed.",
-                time_needed, time_noticed_needed
+                "{time_needed:?} to create this list, {time_noticed_needed:?} to mark all messages as noticed."
             );
         }
         "createchat" => {
@@ -721,26 +739,26 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let contact_id = ContactId::new(arg1.parse()?);
             let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
-            println!("Single#{} created successfully.", chat_id,);
+            println!("Single#{chat_id} created successfully.",);
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
                 chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
-            println!("Group#{} created successfully.", chat_id);
+            println!("Group#{chat_id} created successfully.");
         }
         "createbroadcast" => {
             let chat_id = chat::create_broadcast_list(&context).await?;
 
-            println!("Broadcast#{} created successfully.", chat_id);
+            println!("Broadcast#{chat_id} created successfully.");
         }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
                 chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("Group#{} created and protected successfully.", chat_id);
+            println!("Group#{chat_id} created and protected successfully.");
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
@@ -770,7 +788,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::set_chat_name(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                format!("{} {}", arg1, arg2).trim(),
+                format!("{arg1} {arg2}").trim(),
             )
             .await?;
 
@@ -864,7 +882,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
-                println!("Success, streaming can be stoppped.");
+                println!("Success, streaming can be stopped.");
             }
         }
         "dellocations" => {
@@ -874,7 +892,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No message text given.");
 
-            let msg = format!("{} {}", arg1, arg2);
+            let msg = format!("{arg1} {arg2}");
 
             chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), msg).await?;
         }
@@ -916,7 +934,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
         "sendsyncmsg" => match context.send_sync_msg().await? {
-            Some(msg_id) => println!("sync message sent as {}.", msg_id),
+            Some(msg_id) => println!("sync message sent as {msg_id}."),
             None => println!("sync message not needed."),
         },
         "sendupdate" => {
@@ -936,7 +954,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let query = format!("{} {}", arg1, arg2).trim().to_string();
+            let query = format!("{arg1} {arg2}").trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, &query).await?;
@@ -954,7 +972,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 },
                 query,
             );
-            println!("{:?} to create this list", time_needed);
+            println!("{time_needed:?} to create this list");
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -1000,9 +1018,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
-                    print!("{}", data);
+                    print!("{data}");
                 } else {
-                    print!(", {}", data);
+                    print!(", {data}");
                 }
             }
             println!();
@@ -1073,12 +1091,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(&context, id).await?;
-            println!("{}", res);
+            println!("{res}");
         }
         "download" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            println!("Scheduling download for {:?}", id);
+            println!("Scheduling download for {id:?}");
             id.download_full(&context).await?;
         }
         "html" => {
@@ -1089,7 +1107,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .join(format!("msg-{}.html", id.to_u32()));
             let html = id.get_html(&context).await?.unwrap_or_default();
             fs::write(&file, html).await?;
-            println!("HTML written to: {:#?}", file);
+            println!("HTML written to: {file:#?}");
         }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await?;
@@ -1151,7 +1169,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Arguments [<name>] <addr> expected.");
 
             if !arg2.is_empty() {
-                let book = format!("{}\n{}", arg1, arg2);
+                let book = format!("{arg1}\n{arg2}");
                 Contact::add_address_book(&context, &book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
@@ -1178,10 +1196,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chatlist = Chatlist::try_load(&context, 0, None, Some(contact_id)).await?;
             let chatlist_cnt = chatlist.len();
             if chatlist_cnt > 0 {
-                res += &format!(
-                    "\n\n{} chats shared with Contact#{}: ",
-                    chatlist_cnt, contact_id,
-                );
+                res += &format!("\n\n{chatlist_cnt} chats shared with Contact#{contact_id}: ",);
                 for i in 0..chatlist_cnt {
                     if 0 != i {
                         res += ", ";
@@ -1191,7 +1206,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
 
-            println!("{}", res);
+            println!("{res}");
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
@@ -1215,13 +1230,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let qr = check_qr(&context, arg1).await?;
-            println!("qr={:?}", qr);
+            println!("qr={qr:?}");
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             match set_config_from_qr(&context, arg1).await {
                 Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
-                Err(err) => println!("Cannot set config from QR code: {:?}", err),
+                Err(err) => println!("Cannot set config from QR code: {err:?}"),
             }
         }
         "providerinfo" => {
@@ -1231,7 +1246,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .await?;
             match provider::get_provider_info(&context, arg1, socks5_enabled).await {
                 Some(info) => {
-                    println!("Information for provider belonging to {}:", arg1);
+                    println!("Information for provider belonging to {arg1}:");
                     println!("status: {}", info.status as u32);
                     println!("before_login_hint: {}", info.before_login_hint);
                     println!("after_login_hint: {}", info.after_login_hint);
@@ -1241,7 +1256,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     }
                 }
                 None => {
-                    println!("No information for provider belonging to {} found.", arg1);
+                    println!("No information for provider belonging to {arg1} found.");
                 }
             }
         }
@@ -1250,7 +1265,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             if let Ok(buf) = read_file(&context, &arg1).await {
                 let (width, height) = get_filemeta(&buf)?;
-                println!("width={}, height={}", width, height);
+                println!("width={width}, height={height}");
             } else {
                 bail!("Command failed.");
             }
@@ -1261,8 +1276,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let device_cnt = message::estimate_deletion_cnt(&context, false, seconds).await?;
             let server_cnt = message::estimate_deletion_cnt(&context, true, seconds).await?;
             println!(
-                "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
-                seconds, device_cnt, server_cnt
+                "estimated count of messages older than {seconds} seconds:\non device: {device_cnt}\non server: {server_cnt}"
             );
         }
         "" => (),

deltachat-repl/src/main.rs

@@ -67,8 +67,7 @@ fn receive_event(event: EventType) {
             info!(
                 "{}",
                 yellow.paint(format!(
-                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
-                    chat_id, msg_id,
+                    "Received MSGS_CHANGED(chat_id={chat_id}, msg_id={msg_id})",
                 ))
             );
         }
@@ -80,8 +79,7 @@ fn receive_event(event: EventType) {
             info!(
                 "{}",
                 yellow.paint(format!(
-                    "Received REACTIONS_CHANGED(chat_id={}, msg_id={}, contact_id={})",
-                    chat_id, msg_id, contact_id
+                    "Received REACTIONS_CHANGED(chat_id={chat_id}, msg_id={msg_id}, contact_id={contact_id})"
                 ))
             );
         }
@@ -91,7 +89,7 @@ fn receive_event(event: EventType) {
         EventType::LocationChanged(contact) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+                yellow.paint(format!("Received LOCATION_CHANGED(contact={contact:?})"))
             );
         }
         EventType::ConfigureProgress { progress, comment } => {
@@ -99,21 +97,20 @@ fn receive_event(event: EventType) {
                 info!(
                     "{}",
                     yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
+                        "Received CONFIGURE_PROGRESS({progress} ‰, {comment})"
                     ))
                 );
             } else {
                 info!(
                     "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({progress} ‰)"))
                 );
             }
         }
         EventType::ImexProgress(progress) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+                yellow.paint(format!("Received IMEX_PROGRESS({progress} ‰)"))
             );
         }
         EventType::ImexFileWritten(file) => {
@@ -125,7 +122,7 @@ fn receive_event(event: EventType) {
         EventType::ChatModified(chat) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
+                yellow.paint(format!("Received CHAT_MODIFIED({chat})"))
             );
         }
         _ => {
@@ -155,13 +152,15 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 12] = [
+const IMEX_COMMANDS: [&str; 14] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
     "has-backup",
     "export-backup",
     "import-backup",
+    "send-backup",
+    "receive-backup",
     "export-keys",
     "import-keys",
     "export-setup",
@@ -353,8 +352,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
             match readline {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
-                    rl.add_history_entry(line.as_str());
-                    let contine = Handle::current().block_on(async {
+                    rl.add_history_entry(line.as_str())?;
+                    let should_continue = Handle::current().block_on(async {
                         match handle_cmd(line.trim(), ctx.clone(), &mut selected_chat).await {
                             Ok(ExitResult::Continue) => true,
                             Ok(ExitResult::Exit) => {
@@ -362,13 +361,13 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                                 false
                             }
                             Err(err) => {
-                                println!("Error: {}", err);
+                                println!("Error: {err:#}");
                                 true
                             }
                         }
                     });
 
-                    if !contine {
+                    if !should_continue {
                         break;
                     }
                 }
@@ -377,7 +376,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
                     break;
                 }
                 Err(err) => {
-                    println!("Error: {}", err);
+                    println!("Error: {err:#}");
                     break;
                 }
             }
@@ -444,7 +443,7 @@ async fn handle_cmd(
                 if arg0 == "getbadqr" && qr.len() > 40 {
                     qr.replace_range(12..22, "0000000000")
                 }
-                println!("{}", qr);
+                println!("{qr}");
                 let output = Command::new("qrencode")
                     .args(["-t", "ansiutf8", qr.as_str(), "-o", "-"])
                     .output()
@@ -460,7 +459,7 @@ async fn handle_cmd(
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {
                     fs::write(&file, svg).await?;
-                    println!("QR code svg written to: {:#?}", file);
+                    println!("QR code svg written to: {file:#?}");
                 }
                 Err(err) => {
                     bail!("Failed to get QR code svg: {}", err);

deltachat-rpc-client/examples/echobot_advanced.py

@@ -27,9 +27,7 @@ async def log_error(event):
 
 @hooks.on(events.MemberListChanged)
 async def on_memberlist_changed(event):
-    logging.info(
-        "member %s was %s", event.member, "added" if event.member_added else "removed"
-    )
+    logging.info("member %s was %s", event.member, "added" if event.member_added else "removed")
 
 
 @hooks.on(events.GroupImageChanged)
@@ -66,7 +64,8 @@ async def main():
 
         bot = Bot(account, hooks)
         if not await bot.is_configured():
-            asyncio.create_task(bot.configure(email=sys.argv[1], password=sys.argv[2]))
+            # Save a reference to avoid garbage collection of the task.
+            _configure_task = asyncio.create_task(bot.configure(email=sys.argv[1], password=sys.argv[2]))
         await bot.run_forever()
 
 

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -32,7 +32,7 @@ async def main():
         async def process_messages():
             for message in await account.get_fresh_messages_in_arrival_order():
                 snapshot = await message.get_snapshot()
-                if not snapshot.is_info:
+                if not snapshot.is_bot and not snapshot.is_info:
                     await snapshot.chat.send_text(snapshot.text)
                 await snapshot.message.mark_seen()
 

deltachat-rpc-client/pyproject.toml

@@ -13,13 +13,6 @@ dynamic = [
     "version"
 ]
 
-[tool.setuptools]
-# We declare the package not-zip-safe so that our type hints are also available
-# when checking client code that uses our (installed) package.
-# Ref:
-# https://mypy.readthedocs.io/en/stable/installed_packages.html?highlight=zip#using-installed-packages-with-mypy-pep-561
-zip-safe = false
-
 [tool.setuptools.package-data]
 deltachat_rpc_client = [
     "py.typed"
@@ -27,3 +20,41 @@ deltachat_rpc_client = [
 
 [project.entry-points.pytest11]
 "deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin"
+
+[tool.black]
+line-length = 120
+
+[tool.ruff]
+select = [
+  "E", "W", # pycodestyle
+  "F", # Pyflakes
+  "N", # pep8-naming
+  "I", # isort
+
+  "ARG", # flake8-unused-arguments
+  "B", # flake8-bugbear
+  "C4", # flake8-comprehensions
+  "COM", # flake8-commas
+  "DTZ", # flake8-datetimez
+  "ICN", # flake8-import-conventions
+  "ISC", # flake8-implicit-str-concat
+  "PIE", # flake8-pie
+  "PT", # flake8-pytest-style
+  "RET", # flake8-return
+  "SIM", # flake8-simplify
+  "TCH", # flake8-type-checking
+  "TID", # flake8-tidy-imports
+  "YTT", # flake8-2020
+
+  "ERA", # eradicate
+
+  "PLC", # Pylint Convention
+  "PLE", # Pylint Error
+  "PLW", # Pylint Warning
+
+  "RUF006" # asyncio-dangling-task
+]
+line-length = 120
+
+[tool.isort]
+profile = "black"

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -8,3 +8,18 @@ from .contact import Contact
 from .deltachat import DeltaChat
 from .message import Message
 from .rpc import Rpc
+
+__all__ = [
+    "Account",
+    "AttrDict",
+    "Bot",
+    "Chat",
+    "Client",
+    "Contact",
+    "DeltaChat",
+    "EventType",
+    "Message",
+    "Rpc",
+    "run_bot_cli",
+    "run_client_cli",
+]

deltachat-rpc-client/src/deltachat_rpc_client/_utils.py

@@ -27,15 +27,10 @@ def _to_attrdict(obj):
 
 
 class AttrDict(dict):
-    """Dictionary that allows accessing values usin the "dot notation" as attributes."""
+    """Dictionary that allows accessing values using the "dot notation" as attributes."""
 
     def __init__(self, *args, **kwargs) -> None:
-        super().__init__(
-            {
-                _camel_to_snake(key): _to_attrdict(value)
-                for key, value in dict(*args, **kwargs).items()
-            }
-        )
+        super().__init__({_camel_to_snake(key): _to_attrdict(value) for key, value in dict(*args, **kwargs).items()})
 
     def __getattr__(self, attr):
         if attr in self:
@@ -51,7 +46,7 @@ class AttrDict(dict):
 async def run_client_cli(
     hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
     argv: Optional[list] = None,
-    **kwargs
+    **kwargs,
 ) -> None:
     """Run a simple command line app, using the given hooks.
 
@@ -65,7 +60,7 @@ async def run_client_cli(
 async def run_bot_cli(
     hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
     argv: Optional[list] = None,
-    **kwargs
+    **kwargs,
 ) -> None:
     """Run a simple bot command line using the given hooks.
 
@@ -80,7 +75,7 @@ async def _run_cli(
     client_type: Type["Client"],
     hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
     argv: Optional[list] = None,
-    **kwargs
+    **kwargs,
 ) -> None:
     from .deltachat import DeltaChat
     from .rpc import Rpc
@@ -107,12 +102,10 @@ async def _run_cli(
         client = client_type(account, hooks)
         client.logger.debug("Running deltachat core %s", core_version)
         if not await client.is_configured():
-            assert (
-                args.email and args.password
-            ), "Account is not configured and email and password must be provided"
-            asyncio.create_task(
-                client.configure(email=args.email, password=args.password)
-            )
+            assert args.email, "Account is not configured and email must be provided"
+            assert args.password, "Account is not configured and password must be provided"
+            # Save a reference to avoid garbage collection of the task.
+            _configure_task = asyncio.create_task(client.configure(email=args.email, password=args.password))
         await client.run_forever()
 
 

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 from ._utils import AttrDict
@@ -5,34 +6,23 @@ from .chat import Chat
 from .const import ChatlistFlag, ContactFlag, SpecialContactId
 from .contact import Contact
 from .message import Message
-from .rpc import Rpc
 
 if TYPE_CHECKING:
     from .deltachat import DeltaChat
+    from .rpc import Rpc
 
 
+@dataclass
 class Account:
     """Delta Chat account."""
 
-    def __init__(self, manager: "DeltaChat", account_id: int) -> None:
-        self.manager = manager
-        self.id = account_id
+    manager: "DeltaChat"
+    id: int
 
     @property
-    def _rpc(self) -> Rpc:
+    def _rpc(self) -> "Rpc":
         return self.manager.rpc
 
-    def __eq__(self, other) -> bool:
-        if not isinstance(other, Account):
-            return False
-        return self.id == other.id and self.manager == other.manager
-
-    def __ne__(self, other) -> bool:
-        return not self == other
-
-    def __repr__(self) -> str:
-        return f"<Account id={self.id}>"
-
     async def wait_for_event(self) -> AttrDict:
         """Wait until the next event and return it."""
         return AttrDict(await self._rpc.wait_for_event(self.id))
@@ -89,9 +79,7 @@ class Account:
         """Configure an account."""
         await self._rpc.configure(self.id)
 
-    async def create_contact(
-        self, obj: Union[int, str, Contact], name: Optional[str] = None
-    ) -> Contact:
+    async def create_contact(self, obj: Union[int, str, Contact], name: Optional[str] = None) -> Contact:
         """Create a new Contact or return an existing one.
 
         Calling this method will always result in the same
@@ -120,10 +108,7 @@ class Account:
     async def get_blocked_contacts(self) -> List[AttrDict]:
         """Return a list with snapshots of all blocked contacts."""
         contacts = await self._rpc.get_blocked_contacts(self.id)
-        return [
-            AttrDict(contact=Contact(self, contact["id"]), **contact)
-            for contact in contacts
-        ]
+        return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
 
     async def get_contacts(
         self,
@@ -148,10 +133,7 @@ class Account:
 
         if snapshot:
             contacts = await self._rpc.get_contacts(self.id, flags, query)
-            return [
-                AttrDict(contact=Contact(self, contact["id"]), **contact)
-                for contact in contacts
-            ]
+            return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
         contacts = await self._rpc.get_contact_ids(self.id, flags, query)
         return [Contact(self, contact_id) for contact_id in contacts]
 
@@ -176,7 +158,7 @@ class Account:
         :param contact: if a contact is specified only chats including this contact are returned.
         :param archived_only: if True only archived chats are returned.
         :param for_forwarding: if True the chat list is sorted with "Saved messages" at the top
-                               and withot "Device chat" and contact requests.
+                               and without "Device chat" and contact requests.
         :param no_specials: if True archive link is not added to the list.
         :param alldone_hint: if True the "all done hint" special chat will be added to the list
                              as needed.
@@ -192,9 +174,7 @@ class Account:
         if alldone_hint:
             flags |= ChatlistFlag.ADD_ALLDONE_HINT
 
-        entries = await self._rpc.get_chatlist_entries(
-            self.id, flags, query, contact and contact.id
-        )
+        entries = await self._rpc.get_chatlist_entries(self.id, flags, query, contact and contact.id)
         if not snapshot:
             return [Chat(self, entry[0]) for entry in entries]
 

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -1,39 +1,30 @@
 import calendar
-from datetime import datetime
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
 
 from ._utils import AttrDict
-from .const import ChatVisibility
+from .const import ChatVisibility, ViewType
 from .contact import Contact
 from .message import Message
-from .rpc import Rpc
 
 if TYPE_CHECKING:
+    from datetime import datetime
+
     from .account import Account
+    from .rpc import Rpc
 
 
+@dataclass
 class Chat:
     """Chat object which manages members and through which you can send and retrieve messages."""
 
-    def __init__(self, account: "Account", chat_id: int) -> None:
-        self.account = account
-        self.id = chat_id
+    account: "Account"
+    id: int
 
     @property
-    def _rpc(self) -> Rpc:
+    def _rpc(self) -> "Rpc":
         return self.account._rpc
 
-    def __eq__(self, other) -> bool:
-        if not isinstance(other, Chat):
-            return False
-        return self.id == other.id and self.account == other.account
-
-    def __ne__(self, other) -> bool:
-        return not self == other
-
-    def __repr__(self) -> str:
-        return f"<Chat id={self.id} account={self.account.id}>"
-
     async def delete(self) -> None:
         """Delete this chat and all its messages.
 
@@ -63,7 +54,7 @@ class Chat:
         """
         if duration is not None:
             assert duration > 0, "Invalid duration"
-            dur: Union[str, dict] = dict(Until=duration)
+            dur: Union[str, dict] = {"Until": duration}
         else:
             dur = "Forever"
         await self._rpc.set_chat_mute_duration(self.account.id, self.id, dur)
@@ -74,27 +65,19 @@ class Chat:
 
     async def pin(self) -> None:
         """Pin this chat."""
-        await self._rpc.set_chat_visibility(
-            self.account.id, self.id, ChatVisibility.PINNED
-        )
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.PINNED)
 
     async def unpin(self) -> None:
         """Unpin this chat."""
-        await self._rpc.set_chat_visibility(
-            self.account.id, self.id, ChatVisibility.NORMAL
-        )
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL)
 
     async def archive(self) -> None:
         """Archive this chat."""
-        await self._rpc.set_chat_visibility(
-            self.account.id, self.id, ChatVisibility.ARCHIVED
-        )
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.ARCHIVED)
 
     async def unarchive(self) -> None:
         """Unarchive this chat."""
-        await self._rpc.set_chat_visibility(
-            self.account.id, self.id, ChatVisibility.NORMAL
-        )
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL)
 
     async def set_name(self, name: str) -> None:
         """Set name of this chat."""
@@ -125,17 +108,27 @@ class Chat:
     async def send_message(
         self,
         text: Optional[str] = None,
+        html: Optional[str] = None,
+        viewtype: Optional[ViewType] = None,
         file: Optional[str] = None,
         location: Optional[Tuple[float, float]] = None,
+        override_sender_name: Optional[str] = None,
         quoted_msg: Optional[Union[int, Message]] = None,
     ) -> Message:
         """Send a message and return the resulting Message instance."""
         if isinstance(quoted_msg, Message):
             quoted_msg = quoted_msg.id
 
-        msg_id, _ = await self._rpc.misc_send_msg(
-            self.account.id, self.id, text, file, location, quoted_msg
-        )
+        draft = {
+            "text": text,
+            "html": html,
+            "viewtype": viewtype,
+            "file": file,
+            "location": location,
+            "overrideSenderName": override_sender_name,
+            "quotedMsg": quoted_msg,
+        }
+        msg_id = await self._rpc.send_msg(self.account.id, self.id, draft)
         return Message(self.account, msg_id)
 
     async def send_text(self, text: str) -> Message:
@@ -184,9 +177,9 @@ class Chat:
         snapshot["message"] = Message(self.account, snapshot.id)
         return snapshot
 
-    async def get_messages(self, flags: int = 0) -> List[Message]:
+    async def get_messages(self, info_only: bool = False, add_daymarker: bool = False) -> List[Message]:
         """get the list of messages in this chat."""
-        msgs = await self._rpc.get_message_ids(self.account.id, self.id, flags)
+        msgs = await self._rpc.get_message_ids(self.account.id, self.id, info_only, add_daymarker)
         return [Message(self.account, msg_id) for msg_id in msgs]
 
     async def get_fresh_message_count(self) -> int:
@@ -201,19 +194,23 @@ class Chat:
         """Add contacts to this group."""
         for cnt in contact:
             if isinstance(cnt, str):
-                cnt = (await self.account.create_contact(cnt)).id
+                contact_id = (await self.account.create_contact(cnt)).id
             elif not isinstance(cnt, int):
-                cnt = cnt.id
-            await self._rpc.add_contact_to_chat(self.account.id, self.id, cnt)
+                contact_id = cnt.id
+            else:
+                contact_id = cnt
+            await self._rpc.add_contact_to_chat(self.account.id, self.id, contact_id)
 
     async def remove_contact(self, *contact: Union[int, str, Contact]) -> None:
         """Remove members from this group."""
         for cnt in contact:
             if isinstance(cnt, str):
-                cnt = (await self.account.create_contact(cnt)).id
+                contact_id = (await self.account.create_contact(cnt)).id
             elif not isinstance(cnt, int):
-                cnt = cnt.id
-            await self._rpc.remove_contact_from_chat(self.account.id, self.id, cnt)
+                contact_id = cnt.id
+            else:
+                contact_id = cnt
+            await self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id)
 
     async def get_contacts(self) -> List[Contact]:
         """Get the contacts belonging to this chat.
@@ -237,27 +234,21 @@ class Chat:
     async def get_locations(
         self,
         contact: Optional[Contact] = None,
-        timestamp_from: Optional[datetime] = None,
-        timestamp_to: Optional[datetime] = None,
+        timestamp_from: Optional["datetime"] = None,
+        timestamp_to: Optional["datetime"] = None,
     ) -> List[AttrDict]:
         """Get list of location snapshots for the given contact in the given timespan."""
-        time_from = (
-            calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0
-        )
+        time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0
         time_to = calendar.timegm(timestamp_to.utctimetuple()) if timestamp_to else 0
         contact_id = contact.id if contact else 0
 
-        result = await self._rpc.get_locations(
-            self.account.id, self.id, contact_id, time_from, time_to
-        )
+        result = await self._rpc.get_locations(self.account.id, self.id, contact_id, time_from, time_to)
         locations = []
         contacts: Dict[int, Contact] = {}
         for loc in result:
-            loc = AttrDict(loc)
-            loc["chat"] = self
-            loc["contact"] = contacts.setdefault(
-                loc.contact_id, Contact(self.account, loc.contact_id)
-            )
-            loc["message"] = Message(self.account, loc.msg_id)
-            locations.append(loc)
+            location = AttrDict(loc)
+            location["chat"] = self
+            location["contact"] = contacts.setdefault(location.contact_id, Contact(self.account, location.contact_id))
+            location["message"] = Message(self.account, location.msg_id)
+            locations.append(location)
         return locations

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -2,6 +2,7 @@
 import inspect
 import logging
 from typing import (
+    TYPE_CHECKING,
     Callable,
     Coroutine,
     Dict,
@@ -13,8 +14,6 @@ from typing import (
     Union,
 )
 
-from deltachat_rpc_client.account import Account
-
 from ._utils import (
     AttrDict,
     parse_system_add_remove,
@@ -31,13 +30,16 @@ from .events import (
     RawEvent,
 )
 
+if TYPE_CHECKING:
+    from deltachat_rpc_client.account import Account
+
 
 class Client:
     """Simple Delta Chat client that listen to events of a single account."""
 
     def __init__(
         self,
-        account: Account,
+        account: "Account",
         hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None,
         logger: Optional[logging.Logger] = None,
     ) -> None:
@@ -47,15 +49,11 @@ class Client:
         self._should_process_messages = 0
         self.add_hooks(hooks or [])
 
-    def add_hooks(
-        self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]
-    ) -> None:
+    def add_hooks(self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]) -> None:
         for hook, event in hooks:
             self.add_hook(hook, event)
 
-    def add_hook(
-        self, hook: Callable, event: Union[type, EventFilter] = RawEvent
-    ) -> None:
+    def add_hook(self, hook: Callable, event: Union[type, EventFilter] = RawEvent) -> None:
         """Register hook for the given event filter."""
         if isinstance(event, type):
             event = event()
@@ -64,7 +62,7 @@ class Client:
             isinstance(
                 event,
                 (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
-            )
+            ),
         )
         self._hooks.setdefault(type(event), set()).add((hook, event))
 
@@ -76,7 +74,7 @@ class Client:
             isinstance(
                 event,
                 (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
-            )
+            ),
         )
         self._hooks.get(type(event), set()).remove((hook, event))
 
@@ -95,9 +93,7 @@ class Client:
         """Process events forever."""
         await self.run_until(lambda _: False)
 
-    async def run_until(
-        self, func: Callable[[AttrDict], Union[bool, Coroutine]]
-    ) -> AttrDict:
+    async def run_until(self, func: Callable[[AttrDict], Union[bool, Coroutine]]) -> AttrDict:
         """Process events until the given callable evaluates to True.
 
         The callable should accept an AttrDict object representing the
@@ -122,9 +118,7 @@ class Client:
             if stop:
                 return event
 
-    async def _on_event(
-        self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent
-    ) -> None:
+    async def _on_event(self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent) -> None:
         for hook, evfilter in self._hooks.get(filter_type, []):
             if await evfilter.filter(event):
                 try:
@@ -133,11 +127,7 @@ class Client:
                     self.logger.exception(ex)
 
     async def _parse_command(self, event: AttrDict) -> None:
-        cmds = [
-            hook[1].command
-            for hook in self._hooks.get(NewMessage, [])
-            if hook[1].command
-        ]
+        cmds = [hook[1].command for hook in self._hooks.get(NewMessage, []) if hook[1].command]
         parts = event.message_snapshot.text.split(maxsplit=1)
         payload = parts[1] if len(parts) > 1 else ""
         cmd = parts.pop(0)
@@ -202,11 +192,7 @@ class Client:
             for message in await self.account.get_fresh_messages_in_arrival_order():
                 snapshot = await message.get_snapshot()
                 await self._on_new_msg(snapshot)
-                if (
-                    snapshot.is_info
-                    and snapshot.system_message_type
-                    != SystemMessageType.WEBXDC_INFO_MESSAGE
-                ):
+                if snapshot.is_info and snapshot.system_message_type != SystemMessageType.WEBXDC_INFO_MESSAGE:
                     await self._handle_info_msg(snapshot)
                 await snapshot.message.mark_seen()
 

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -1,13 +1,15 @@
+from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 from ._utils import AttrDict
-from .rpc import Rpc
 
 if TYPE_CHECKING:
     from .account import Account
     from .chat import Chat
+    from .rpc import Rpc
 
 
+@dataclass
 class Contact:
     """
     Contact API.
@@ -15,23 +17,11 @@ class Contact:
     Essentially a wrapper for RPC, account ID and a contact ID.
     """
 
-    def __init__(self, account: "Account", contact_id: int) -> None:
-        self.account = account
-        self.id = contact_id
-
-    def __eq__(self, other) -> bool:
-        if not isinstance(other, Contact):
-            return False
-        return self.id == other.id and self.account == other.account
-
-    def __ne__(self, other) -> bool:
-        return not self == other
-
-    def __repr__(self) -> str:
-        return f"<Contact id={self.id} account={self.account.id}>"
+    account: "Account"
+    id: int
 
     @property
-    def _rpc(self) -> Rpc:
+    def _rpc(self) -> "Rpc":
         return self.account._rpc
 
     async def block(self) -> None:

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -1,8 +1,10 @@
-from typing import Dict, List
+from typing import TYPE_CHECKING, Dict, List
 
 from ._utils import AttrDict
 from .account import Account
-from .rpc import Rpc
+
+if TYPE_CHECKING:
+    from .rpc import Rpc
 
 
 class DeltaChat:
@@ -11,7 +13,7 @@ class DeltaChat:
     This is the root of the object oriented API.
     """
 
-    def __init__(self, rpc: Rpc) -> None:
+    def __init__(self, rpc: "Rpc") -> None:
         self.rpc = rpc
 
     async def add_account(self) -> Account:

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -2,15 +2,17 @@
 import inspect
 import re
 from abc import ABC, abstractmethod
-from typing import Callable, Iterable, Iterator, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union
 
-from ._utils import AttrDict
 from .const import EventType
 
+if TYPE_CHECKING:
+    from ._utils import AttrDict
+
 
 def _tuple_of(obj, type_: type) -> tuple:
     if not obj:
-        return tuple()
+        return ()
     if isinstance(obj, type_):
         obj = (obj,)
 
@@ -39,7 +41,7 @@ class EventFilter(ABC):
         """Return True if two event filters are equal."""
 
     def __ne__(self, other):
-        return not self.__eq__(other)
+        return not self == other
 
     async def _call_func(self, event) -> bool:
         if not self.func:
@@ -65,9 +67,7 @@ class RawEvent(EventFilter):
                  should be dispatched or not.
     """
 
-    def __init__(
-        self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs
-    ):
+    def __init__(self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs):
         super().__init__(**kwargs)
         try:
             self.types = _tuple_of(types, EventType)
@@ -82,7 +82,7 @@ class RawEvent(EventFilter):
             return (self.types, self.func) == (other.types, other.func)
         return False
 
-    async def filter(self, event: AttrDict) -> bool:
+    async def filter(self, event: "AttrDict") -> bool:
         if self.types and event.type not in self.types:
             return False
         return await self._call_func(event)
@@ -98,6 +98,9 @@ class NewMessage(EventFilter):
                     content.
     :param command: If set, only match messages with the given command (ex. /help).
                     Setting this property implies `is_info==False`.
+    :param is_bot: If set to True only match messages sent by bots, if set to None
+                   match messages from bots and users. If omitted or set to False
+                   only messages from users will be matched.
     :param is_info: If set to True only match info/system messages, if set to False
                     only match messages that are not info/system messages. If omitted
                     info/system messages as well as normal messages will be matched.
@@ -115,10 +118,12 @@ class NewMessage(EventFilter):
             re.Pattern,
         ] = None,
         command: Optional[str] = None,
+        is_bot: Optional[bool] = False,
         is_info: Optional[bool] = None,
-        func: Optional[Callable[[AttrDict], bool]] = None,
+        func: Optional[Callable[["AttrDict"], bool]] = None,
     ) -> None:
         super().__init__(func=func)
+        self.is_bot = is_bot
         self.is_info = is_info
         if command is not None and not isinstance(command, str):
             raise TypeError("Invalid command")
@@ -135,19 +140,28 @@ class NewMessage(EventFilter):
             raise TypeError("Invalid pattern type")
 
     def __hash__(self) -> int:
-        return hash((self.pattern, self.func))
+        return hash((self.pattern, self.command, self.is_bot, self.is_info, self.func))
 
     def __eq__(self, other) -> bool:
         if isinstance(other, NewMessage):
-            return (self.pattern, self.command, self.is_info, self.func) == (
+            return (
+                self.pattern,
+                self.command,
+                self.is_bot,
+                self.is_info,
+                self.func,
+            ) == (
                 other.pattern,
                 other.command,
+                other.is_bot,
                 other.is_info,
                 other.func,
             )
         return False
 
-    async def filter(self, event: AttrDict) -> bool:
+    async def filter(self, event: "AttrDict") -> bool:
+        if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot:
+            return False
         if self.is_info is not None and self.is_info != event.message_snapshot.is_info:
             return False
         if self.command and self.command != event.command:
@@ -187,7 +201,7 @@ class MemberListChanged(EventFilter):
             return (self.added, self.func) == (other.added, other.func)
         return False
 
-    async def filter(self, event: AttrDict) -> bool:
+    async def filter(self, event: "AttrDict") -> bool:
         if self.added is not None and self.added != event.member_added:
             return False
         return await self._call_func(event)
@@ -219,7 +233,7 @@ class GroupImageChanged(EventFilter):
             return (self.deleted, self.func) == (other.deleted, other.func)
         return False
 
-    async def filter(self, event: AttrDict) -> bool:
+    async def filter(self, event: "AttrDict") -> bool:
         if self.deleted is not None and self.deleted != event.image_deleted:
             return False
         return await self._call_func(event)
@@ -244,7 +258,7 @@ class GroupNameChanged(EventFilter):
             return self.func == other.func
         return False
 
-    async def filter(self, event: AttrDict) -> bool:
+    async def filter(self, event: "AttrDict") -> bool:
         return await self._call_func(event)
 
 

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -1,34 +1,24 @@
 import json
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Union
 
 from ._utils import AttrDict
 from .contact import Contact
-from .rpc import Rpc
 
 if TYPE_CHECKING:
     from .account import Account
+    from .rpc import Rpc
 
 
+@dataclass
 class Message:
     """Delta Chat Message object."""
 
-    def __init__(self, account: "Account", msg_id: int) -> None:
-        self.account = account
-        self.id = msg_id
-
-    def __eq__(self, other) -> bool:
-        if not isinstance(other, Message):
-            return False
-        return self.id == other.id and self.account == other.account
-
-    def __ne__(self, other) -> bool:
-        return not self == other
-
-    def __repr__(self) -> str:
-        return f"<Message id={self.id} account={self.account.id}>"
+    account: "Account"
+    id: int
 
     @property
-    def _rpc(self) -> Rpc:
+    def _rpc(self) -> "Rpc":
         return self.account._rpc
 
     async def send_reaction(self, *reaction: str):
@@ -49,22 +39,14 @@ class Message:
         """Mark the message as seen."""
         await self._rpc.markseen_msgs(self.account.id, [self.id])
 
-    async def send_webxdc_status_update(
-        self, update: Union[dict, str], description: str
-    ) -> None:
+    async def send_webxdc_status_update(self, update: Union[dict, str], description: str) -> None:
         """Send a webxdc status update. This message must be a webxdc."""
         if not isinstance(update, str):
             update = json.dumps(update)
-        await self._rpc.send_webxdc_status_update(
-            self.account.id, self.id, update, description
-        )
+        await self._rpc.send_webxdc_status_update(self.account.id, self.id, update, description)
 
     async def get_webxdc_status_updates(self, last_known_serial: int = 0) -> list:
-        return json.loads(
-            await self._rpc.get_webxdc_status_updates(
-                self.account.id, self.id, last_known_serial
-            )
-        )
+        return json.loads(await self._rpc.get_webxdc_status_updates(self.account.id, self.id, last_known_serial))
 
     async def get_webxdc_info(self) -> dict:
         return await self._rpc.get_webxdc_info(self.account.id, self.id)

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import os
 from typing import AsyncGenerator, List, Optional
@@ -51,11 +52,13 @@ class ACFactory:
         await bot.configure(credentials["email"], credentials["password"])
         return bot
 
+    async def get_online_account(self) -> Account:
+        account = await self.new_configured_account()
+        await account.start_io()
+        return account
+
     async def get_online_accounts(self, num: int) -> List[Account]:
-        accounts = [await self.new_configured_account() for _ in range(num)]
-        for account in accounts:
-            await account.start_io()
-        return accounts
+        return await asyncio.gather(*[self.get_online_account() for _ in range(num)])
 
     async def send_message(
         self,
@@ -67,9 +70,7 @@ class ACFactory:
     ) -> Message:
         if not from_account:
             from_account = (await self.get_online_accounts(1))[0]
-        to_contact = await from_account.create_contact(
-            await to_account.get_config("addr")
-        )
+        to_contact = await from_account.create_contact(await to_account.get_config("addr"))
         if group:
             to_chat = await from_account.create_group(group)
             await to_chat.add_contact(to_contact)

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -30,7 +30,7 @@ class Rpc:
             "deltachat-rpc-server",
             stdin=asyncio.subprocess.PIPE,
             stdout=asyncio.subprocess.PIPE,
-            **self._kwargs
+            **self._kwargs,
         )
         self.id = 0
         self.event_queues = {}
@@ -46,7 +46,7 @@ class Rpc:
         await self.start()
         return self
 
-    async def __aexit__(self, exc_type, exc, tb):
+    async def __aexit__(self, _exc_type, _exc, _tb):
         await self.close()
 
     async def reader_loop(self) -> None:
@@ -97,5 +97,6 @@ class Rpc:
                 raise JsonRpcError(response["error"])
             if "result" in response:
                 return response["result"]
+            return None
 
         return method

deltachat-rpc-client/tests/test_something.py

@@ -1,19 +1,30 @@
+import asyncio
 from unittest.mock import MagicMock
 
 import pytest
-
 from deltachat_rpc_client import EventType, events
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_system_info(rpc) -> None:
     system_info = await rpc.get_system_info()
     assert "arch" in system_info
     assert "deltachat_core_version" in system_info
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
+async def test_sleep(rpc) -> None:
+    """Test that long-running task does not block short-running task from completion."""
+    sleep_5_task = asyncio.create_task(rpc.sleep(5.0))
+    sleep_3_task = asyncio.create_task(rpc.sleep(3.0))
+    done, pending = await asyncio.wait([sleep_5_task, sleep_3_task], return_when=asyncio.FIRST_COMPLETED)
+    assert sleep_3_task in done
+    assert sleep_5_task in pending
+    sleep_5_task.cancel()
+
+
+@pytest.mark.asyncio()
 async def test_email_address_validity(rpc) -> None:
     valid_addresses = [
         "email@example.com",
@@ -27,7 +38,7 @@ async def test_email_address_validity(rpc) -> None:
         assert not await rpc.check_email_validity(addr)
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_acfactory(acfactory) -> None:
     account = await acfactory.new_configured_account()
     while True:
@@ -41,17 +52,18 @@ async def test_acfactory(acfactory) -> None:
     print("Successful configuration")
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_configure_starttls(acfactory) -> None:
     account = await acfactory.new_preconfigured_account()
 
     # Use STARTTLS
     await account.set_config("mail_security", "2")
+    await account.set_config("send_security", "2")
     await account.configure()
     assert await account.is_configured()
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_account(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
 
@@ -111,7 +123,7 @@ async def test_account(acfactory) -> None:
     await alice.stop_io()
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_chat(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
 
@@ -177,7 +189,7 @@ async def test_chat(acfactory) -> None:
     await group.get_locations()
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_contact(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
 
@@ -195,7 +207,7 @@ async def test_contact(acfactory) -> None:
     await alice_contact_bob.create_chat()
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_message(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
 
@@ -215,6 +227,7 @@ async def test_message(acfactory) -> None:
     snapshot = await message.get_snapshot()
     assert snapshot.chat_id == chat_id
     assert snapshot.text == "Hello!"
+    assert not snapshot.is_bot
     assert repr(message)
 
     with pytest.raises(JsonRpcError):  # chat is not accepted
@@ -226,44 +239,67 @@ async def test_message(acfactory) -> None:
     await message.send_reaction("😎")
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
+async def test_is_bot(acfactory) -> None:
+    """Test that we can recognize messages submitted by bots."""
+    alice, bob = await acfactory.get_online_accounts(2)
+
+    bob_addr = await bob.get_config("addr")
+    alice_contact_bob = await alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = await alice_contact_bob.create_chat()
+
+    # Alice becomes a bot.
+    await alice.set_config("bot", "1")
+    await alice_chat_bob.send_text("Hello!")
+
+    while True:
+        event = await bob.wait_for_event()
+        if event.type == EventType.INCOMING_MSG:
+            msg_id = event.msg_id
+            message = bob.get_message_by_id(msg_id)
+            snapshot = await message.get_snapshot()
+            assert snapshot.chat_id == event.chat_id
+            assert snapshot.text == "Hello!"
+            assert snapshot.is_bot
+            break
+
+
+@pytest.mark.asyncio()
 async def test_bot(acfactory) -> None:
     mock = MagicMock()
     user = (await acfactory.get_online_accounts(1))[0]
     bot = await acfactory.new_configured_bot()
+    bot2 = await acfactory.new_configured_bot()
 
     assert await bot.is_configured()
     assert await bot.account.get_config("bot") == "1"
 
-    hook = lambda e: mock.hook(e.msg_id), events.RawEvent(EventType.INCOMING_MSG)
+    hook = lambda e: mock.hook(e.msg_id) and None, events.RawEvent(EventType.INCOMING_MSG)
     bot.add_hook(*hook)
-    event = await acfactory.process_message(
-        from_account=user, to_client=bot, text="Hello!"
-    )
+    event = await acfactory.process_message(from_account=user, to_client=bot, text="Hello!")
+    snapshot = await bot.account.get_message_by_id(event.msg_id).get_snapshot()
+    assert not snapshot.is_bot
     mock.hook.assert_called_once_with(event.msg_id)
     bot.remove_hook(*hook)
 
-    track = lambda e: mock.hook(e.message_snapshot.id)
+    def track(e):
+        mock.hook(e.message_snapshot.id)
 
     mock.hook.reset_mock()
     hook = track, events.NewMessage(r"hello")
     bot.add_hook(*hook)
     bot.add_hook(track, events.NewMessage(command="/help"))
-    event = await acfactory.process_message(
-        from_account=user, to_client=bot, text="hello"
-    )
+    event = await acfactory.process_message(from_account=user, to_client=bot, text="hello")
     mock.hook.assert_called_with(event.msg_id)
-    event = await acfactory.process_message(
-        from_account=user, to_client=bot, text="hello!"
-    )
+    event = await acfactory.process_message(from_account=user, to_client=bot, text="hello!")
     mock.hook.assert_called_with(event.msg_id)
+    await acfactory.process_message(from_account=bot2.account, to_client=bot, text="hello")
+    assert len(mock.hook.mock_calls) == 2  # bot messages are ignored between bots
     await acfactory.process_message(from_account=user, to_client=bot, text="hey!")
     assert len(mock.hook.mock_calls) == 2
     bot.remove_hook(*hook)
 
     mock.hook.reset_mock()
     await acfactory.process_message(from_account=user, to_client=bot, text="hello")
-    event = await acfactory.process_message(
-        from_account=user, to_client=bot, text="/help"
-    )
+    event = await acfactory.process_message(from_account=user, to_client=bot, text="/help")
     mock.hook.assert_called_once_with(event.msg_id)

deltachat-rpc-client/tests/test_webxdc.py

@@ -1,18 +1,15 @@
 import pytest
-
 from deltachat_rpc_client import EventType
 
 
-@pytest.mark.asyncio
+@pytest.mark.asyncio()
 async def test_webxdc(acfactory) -> None:
     alice, bob = await acfactory.get_online_accounts(2)
 
     bob_addr = await bob.get_config("addr")
     alice_contact_bob = await alice.create_contact(bob_addr, "Bob")
     alice_chat_bob = await alice_contact_bob.create_chat()
-    await alice_chat_bob.send_message(
-        text="Let's play chess!", file="../test-data/webxdc/chess.xdc"
-    )
+    await alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
 
     while True:
         event = await bob.wait_for_event()

deltachat-rpc-client/tox.ini

@@ -2,10 +2,11 @@
 isolated_build = true
 envlist =
     py3
+    lint
 
 [testenv]
 commands =
-    pytest {posargs}
+    pytest --exitfirst {posargs}
 setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
     RUST_MIN_STACK=8388608
@@ -16,3 +17,13 @@ deps =
     pytest-asyncio
     aiohttp
     aiodns
+
+[testenv:lint]
+skipsdist = True
+skip_install = True
+deps =
+    ruff
+    black
+commands =
+    black --quiet --check --diff src/ examples/ tests/
+    ruff src/ examples/ tests/

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.105.0"
+version = "1.111.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -13,7 +13,7 @@ categories = ["cryptography", "std", "email"]
 name = "deltachat-rpc-server"
 
 [dependencies]
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc" }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
 
 anyhow = "1"
 env_logger = { version = "0.10.0" }
@@ -21,5 +21,9 @@ futures-lite = "1.12.0"
 log = "0.4"
 serde_json = "1.0.91"
 serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.23.1", features = ["io-std"] }
-yerpc = { version = "0.3.1", features = ["anyhow_expose"] }
+tokio = { version = "1.25.0", features = ["io-std"] }
+yerpc = { version = "0.4.0", features = ["anyhow_expose"] }
+
+[features]
+default = ["vendored"]
+vendored = ["deltachat-jsonrpc/vendored"]

deltachat-rpc-server/README.md

@@ -0,0 +1,34 @@
+# Delta Chat RPC server
+
+This program provides a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) interface to DeltaChat
+over standard I/O.
+
+## Install
+
+To download binary pre-builds check the [releases page](https://github.com/deltachat/deltachat-core-rust/releases).
+Rename the downloaded binary to `deltachat-rpc-server` and add it to your `PATH`.
+
+To install from source run:
+
+```sh
+cargo install --git https://github.com/deltachat/deltachat-core-rust/ deltachat-rpc-server
+```
+
+The `deltachat-rpc-server` executable will be installed into `$HOME/.cargo/bin` that should be available
+in your `PATH`.
+
+## Usage
+
+To use just run `deltachat-rpc-server` command. The accounts folder will be created in the current
+working directory unless `DC_ACCOUNTS_PATH` is set:
+
+```sh
+export DC_ACCOUNTS_PATH=$HOME/delta/
+deltachat-rpc-server
+```
+
+The common use case for this program is to create bindings to use Delta Chat core from programming
+languages other than Rust, for example:
+
+1. Python: https://github.com/deltachat/deltachat-core-rust/tree/master/deltachat-rpc-client/
+2. Go: https://github.com/deltachat/deltachat-rpc-client-go/

deltachat-rpc-server/src/bin/deltachat-rpc-server/main.rs

@@ -40,7 +40,7 @@ async fn main() -> Result<()> {
         while let Some(message) = out_receiver.next().await {
             let message = serde_json::to_string(&message)?;
             log::trace!("RPC send {}", message);
-            println!("{}", message);
+            println!("{message}");
         }
         Ok(())
     });
@@ -51,7 +51,10 @@ async fn main() -> Result<()> {
         let mut lines = BufReader::new(stdin).lines();
         while let Some(message) = lines.next_line().await? {
             log::trace!("RPC recv {}", message);
-            session.handle_incoming(&message).await;
+            let session = session.clone();
+            tokio::spawn(async move {
+                session.handle_incoming(&message).await;
+            });
         }
         log::info!("EOF reached on stdin");
         Ok(())

deltachat_derive/src/lib.rs

@@ -1,10 +1,11 @@
 #![recursion_limit = "128"]
 extern crate proc_macro;
 
-use crate::proc_macro::TokenStream;
 use quote::quote;
 
-// For now, assume (not check) that these macroses are applied to enum without
+use crate::proc_macro::TokenStream;
+
+// For now, assume (not check) that these macros are applied to enum without
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 

deny.toml

@@ -0,0 +1,84 @@
+[advisories]
+unmaintained = "allow"
+ignore = [
+    "RUSTSEC-2020-0071",
+
+    # Only affects windows if using non-default allocator (and unmaintained).
+    "RUSTSEC-2021-0145",
+]
+
+[bans]
+# Accept some duplicate versions, ideally we work towards this list
+# becoming empty.  Adding versions forces us to revisit this at least
+# when upgrading.
+skip = [
+     { name = "base64", version = "<0.21" },
+     { name = "block-buffer", version = "<0.10" },
+     { name = "darling", version = "<0.14" },
+     { name = "darling_core", version = "<0.14" },
+     { name = "darling_macro", version = "<0.14" },
+     { name = "digest", version = "<0.10" },
+     { name = "env_logger", version = "<0.10" },
+     { name = "getrandom", version = "<0.2" },
+     { name = "hermit-abi", version = "<0.3" },
+     { name = "humantime", version = "<2.1" },
+     { name = "idna", version = "<0.3" },
+     { name = "nom", version = "<7.1" },
+     { name = "quick-error", version = "<2.0" },
+     { name = "rand", version = "<0.8" },
+     { name = "rand_chacha", version = "<0.3" },
+     { name = "rand_core", version = "<0.6" },
+     { name = "sha2", version = "<0.10" },
+     { name = "time", version = "<0.3" },
+     { name = "version_check", version = "<0.9" },
+     { name = "wasi", version = "<0.11" },
+     { name = "windows-sys", version = "<0.45" },
+     { name = "windows_x86_64_msvc", version = "<0.42" },
+     { name = "windows_x86_64_gnu", version = "<0.42" },
+     { name = "windows_i686_msvc", version = "<0.42" },
+     { name = "windows_i686_gnu", version = "<0.42" },
+     { name = "windows_aarch64_msvc", version = "<0.42" },
+     { name = "unicode-xid", version = "<0.2.4" },
+     { name = "syn", version = "<1.0" },
+     { name = "quote", version = "<1.0" },
+     { name = "proc-macro2", version = "<1.0" },
+     { name = "portable-atomic", version = "<1.0" },
+     { name = "spin", version = "<0.9.6" },
+     { name = "convert_case", version = "0.4.0" },
+     { name = "clap_lex", version = "0.2.4" },
+     { name = "clap", version = "3.2.23" },
+]
+
+
+[licenses]
+allow = [
+    "0BSD",
+    "Apache-2.0",
+    "BSD-2-Clause",
+    "BSD-3-Clause",
+    "BSL-1.0", # Boost Software License 1.0
+    "CC0-1.0",
+    "ISC",
+    "MIT",
+    "MPL-2.0",
+    "OpenSSL",
+    "Unicode-DFS-2016",
+    "Zlib",
+]
+
+[[licenses.clarify]]
+name = "ring"
+expression = "MIT AND ISC AND OpenSSL"
+license-files = [
+      { path = "LICENSE", hash = 0xbd0eed23 },
+]
+
+[sources.allow-org]
+# Organisations which we allow git sources from.
+github = [
+       "async-email",
+       "deltachat",
+       "n0-computer",
+       "quinn-rs",
+       "dignifiedquire",
+]

draft/aeap-mvp.md

@@ -66,13 +66,13 @@ Note that usually a mail is signed by a key that has a UID matching the from add
 
 #### Upsides:
 - With this approach, it's easy to switch to a model where the info about the transition is encoded in the PGP key. Since the key is gossiped, the information about the transition will spread virally.
-- Faster transation: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.
+- Faster transition: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.
 
 ### Alternatives and old discussions/plans:
 
 - Change the contact instead of rewriting the group member lists. This seems to call for more trouble since we will end up with multiple contacts having the same email address.
 
-- If needed, we could add a header a) indicating that the sender did an address transition or b) listing all the secondary (old) addresses.  For now, there is no big enough benefit to warrant introducing another header and its processing on the receiver side (including all the neccessary checks and handling of error cases). Instead, we only check for the `Chat-Version` header to prevent accidental transitions when an MUA user sends a message from another email address with the same key.
+- If needed, we could add a header a) indicating that the sender did an address transition or b) listing all the secondary (old) addresses.  For now, there is no big enough benefit to warrant introducing another header and its processing on the receiver side (including all the necessary checks and handling of error cases). Instead, we only check for the `Chat-Version` header to prevent accidental transitions when an MUA user sends a message from another email address with the same key.
 
 - The condition for a transition temporarily was:
 
@@ -107,7 +107,7 @@ The most obvious alternative would be to create a new contact with the new addre
 
 #### Upsides:
 - With this approach, it's easier to switch to a model where the info about the transition is encoded in the PGP key. Since the key is gossiped, the information about the transition will spread virally.
-- (Also, less important: Slightly faster transation: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.)
+- (Also, less important: Slightly faster transition: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.)
 - It's easier to implement (if too many problems turn up, we can still switch to another approach and didn't wast that much development time.)
 
 [full messages](https://github.com/deltachat/deltachat-core-rust/pull/2896#discussion_r852002161)
@@ -124,4 +124,4 @@ Other
 Notes during implementing
 ========================
 
-- As far as I understand the code, unencrypted messages are unsigned. So, the transition only works if both sides have the other side's key.
+- As far as I understand the code, unencrypted messages are unsigned. So, the transition only works if both sides have the other side's key.

draft/msgwork_new_imap_jobs.rst

@@ -62,5 +62,5 @@ Notes/Questions:
 
   and design such that we can cover all imap flags.
 
-- It might not be neccessary to keep needs_send_mdn state in this table
+- It might not be necessary to keep needs_send_mdn state in this table
   if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/simple.rs

@@ -1,5 +1,3 @@
-use tempfile::tempdir;
-
 use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::*;
 use deltachat::config;
@@ -8,6 +6,7 @@ use deltachat::context::*;
 use deltachat::message::Message;
 use deltachat::stock_str::StockStrings;
 use deltachat::{EventType, Events};
+use tempfile::tempdir;
 
 fn cb(event: EventType) {
     match event {
@@ -29,7 +28,7 @@ fn cb(event: EventType) {
     }
 }
 
-/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple -- email pw`.
 #[tokio::main]
 async fn main() {
     pretty_env_logger::try_init_timed().ok();
@@ -75,7 +74,7 @@ async fn main() {
 
     for i in 0..1 {
         log::info!("sending message {}", i);
-        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
+        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {i}nth message!"))
             .await
             .unwrap();
     }

fuzz/Cargo.lock

@@ -111,7 +111,7 @@ version = "0.6.0"
 source = "git+https://github.com/async-email/async-imap?branch=master#85ff7a3d9d71a3715354fabf2fc1a8d047b5710e"
 dependencies = [
  "async-channel",
- "async-native-tls",
+ "async-native-tls 0.4.0",
  "base64 0.13.1",
  "byte-pool",
  "chrono",
@@ -140,22 +140,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "async-smtp"
+name = "async-native-tls"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6da21e1dd19fbad3e095ad519fb1558ab77fd82e5c4778dca8f9be0464589e1e"
+checksum = "9343dc5acf07e79ff82d0c37899f079db3534d99f189a1837c8e549c99405bec"
 dependencies = [
- "async-native-tls",
- "async-trait",
+ "native-tls",
+ "thiserror",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "async-smtp"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7384febcabdd07a498c9f4fbaa7e488ff4eb60d0ade14b47b09ec44b8f645301"
+dependencies = [
+ "anyhow",
  "base64 0.13.1",
  "bufstream",
- "fast-socks5",
  "futures",
  "hostname",
  "log",
  "nom 7.1.1",
  "pin-project",
- "pin-utils",
  "thiserror",
  "tokio",
 ]
@@ -235,9 +244,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
 [[package]]
 name = "base64"
-version = "0.20.0"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
 
 [[package]]
 name = "base64ct"
@@ -699,16 +708,16 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
 
 [[package]]
 name = "deltachat"
-version = "1.104.0"
+version = "1.111.0"
 dependencies = [
  "anyhow",
  "async-channel",
  "async-imap",
- "async-native-tls",
+ "async-native-tls 0.5.0",
  "async-smtp",
  "async_zip",
  "backtrace",
- "base64 0.20.0",
+ "base64 0.21.0",
  "bitflags",
  "chrono",
  "deltachat_derive",
@@ -726,18 +735,17 @@ dependencies = [
  "lettre_email",
  "libc",
  "mailparse 0.14.0",
- "native-tls",
  "num-derive",
  "num-traits",
  "num_cpus",
  "once_cell",
+ "parking_lot",
  "percent-encoding",
  "pgp",
  "qrcodegen",
  "quick-xml",
- "r2d2",
- "r2d2_sqlite",
  "rand 0.8.5",
+ "ratelimit",
  "regex",
  "reqwest",
  "rusqlite",
@@ -1286,15 +1294,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "hashbrown"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
-dependencies = [
- "ahash",
-]
-
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -1306,11 +1305,11 @@ dependencies = [
 
 [[package]]
 name = "hashlink"
-version = "0.7.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf"
+checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa"
 dependencies = [
- "hashbrown 0.11.2",
+ "hashbrown",
 ]
 
 [[package]]
@@ -1508,7 +1507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
 dependencies = [
  "autocfg",
- "hashbrown 0.12.3",
+ "hashbrown",
 ]
 
 [[package]]
@@ -1633,9 +1632,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.24.2"
+version = "0.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "898745e570c7d0453cc1fbc4a701eb6c662ed54e8fec8b7d14be137ebeeb9d14"
+checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
 dependencies = [
  "cc",
  "openssl-sys",
@@ -1812,6 +1811,15 @@ dependencies = [
  "minimal-lexical",
 ]
 
+[[package]]
+name = "nom8"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "num-bigint-dig"
 version = "0.8.2"
@@ -1948,9 +1956,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.24.0+1.1.1s"
+version = "111.25.0+1.1.1t"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd"
+checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6"
 dependencies = [
  "cc",
 ]
@@ -2234,27 +2242,6 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20f14e071918cbeefc5edc986a7aa92c425dae244e003a35e1cdddb5ca39b5cb"
 
-[[package]]
-name = "r2d2"
-version = "0.8.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93"
-dependencies = [
- "log",
- "parking_lot",
- "scheduled-thread-pool",
-]
-
-[[package]]
-name = "r2d2_sqlite"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fdc8e4da70586127893be32b7adf21326a4c6b1aba907611edf467d13ffe895"
-dependencies = [
- "r2d2",
- "rusqlite",
-]
-
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -2326,6 +2313,10 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "ratelimit"
+version = "1.0.0"
+
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -2363,11 +2354,11 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.13"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c"
+checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9"
 dependencies = [
- "base64 0.13.1",
+ "base64 0.21.0",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -2440,16 +2431,15 @@ dependencies = [
 
 [[package]]
 name = "rusqlite"
-version = "0.27.0"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a"
+checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a"
 dependencies = [
  "bitflags",
  "fallible-iterator",
  "fallible-streaming-iterator",
  "hashlink",
  "libsqlite3-sys",
- "memchr",
  "smallvec",
 ]
 
@@ -2503,15 +2493,6 @@ dependencies = [
  "windows-sys 0.36.1",
 ]
 
-[[package]]
-name = "scheduled-thread-pool"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "977a7519bff143a44f842fd07e80ad1329295bd71686457f18e496736f4bf9bf"
-dependencies = [
- "parking_lot",
-]
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -2578,6 +2559,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_spanned"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_urlencoded"
 version = "0.7.1"
@@ -2863,9 +2853,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.24.1"
+version = "1.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae"
+checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af"
 dependencies = [
  "autocfg",
  "bytes",
@@ -2952,11 +2942,36 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.5.10"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f"
+checksum = "f7afcae9e3f0fe2c370fd4657108972cbb2fa9db1b9f84849cefd80741b01cb6"
 dependencies = [
  "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.19.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e6a7712b49e1775fb9a7b998de6635b299237f48b404dde71704f2e0e7f37e5"
+dependencies = [
+ "indexmap",
+ "nom8",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
 ]
 
 [[package]]
@@ -3081,7 +3096,7 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137"
 dependencies = [
- "hashbrown 0.12.3",
+ "hashbrown",
  "regex",
 ]
 

node/README.md

@@ -28,7 +28,7 @@ This code used to live at [`deltachat-node`](https://github.com/deltachat/deltac
 
 ## Install
 
-By default the installation will build try to use the bundled prebuilds in the
+By default the installation will try to use the bundled prebuilds in the
 npm package. If this fails it falls back to compile `../deltachat-core-rust` from
 this repository, using `scripts/rebuild-core.js`.
 
@@ -124,7 +124,7 @@ $ npm run test
 ```
 
 (when using [fnm](https://github.com/Schniz/fnm) instead of nvm, you can select the architecture)
-If your node and electron are already build for arm64 you can also try bulding for arm:
+If your node and electron are already build for arm64 you can also try building for arm:
 
 ```
 $ fnm install 16 --arch arm64
@@ -182,7 +182,7 @@ this example can also be found in the examples folder [examples/send_message.js]
 
 ### Generating Docs
 
-We are curently migrating to automaticaly generated documentation.
+We are currently migrating to automatically generated documentation.
 You can find the old documentation at [old_docs](./old_docs).
 
 to generate the documentation, run:

node/lib/binding.ts

@@ -2,7 +2,7 @@ import { join } from 'path'
 
 /**
  * bindings are not typed yet.
- * if the availible function names are required they can be found inside of `../src/module.c`
+ * if the available function names are required they can be found inside of `../src/module.c`
  */
 export const bindings: any = require('node-gyp-build')(join(__dirname, '../'))
 

node/lib/context.ts

@@ -543,7 +543,7 @@ export class Context extends EventEmitter {
 
   /**
    *
-   * @deprectated please use `AccountManager.getSystemInfo()` instead
+   * @deprecated please use `AccountManager.getSystemInfo()` instead
    */
   static getSystemInfo() {
     return AccountManager.getSystemInfo()
@@ -818,7 +818,7 @@ export class Context extends EventEmitter {
    *     Must be given in number of seconds since 00:00 hours, Jan 1, 1970 UTC.
    *     0 for "all up to now".
    * @return Array of locations, NULL is never returned.
-   *     The array is sorted decending;
+   *     The array is sorted descending;
    *     the first entry in the array is the location with the newest timestamp.
    *
    * Examples:

node/lib/deltachat.ts

@@ -217,7 +217,7 @@ export class AccountManager extends EventEmitter {
   /** get information about the provider
    *
    * This function creates a temporary context to be standalone,
-   * if posible use `Context.getProviderFromEmail` instead. (otherwise potential proxy settings are not used)
+   * if possible use `Context.getProviderFromEmail` instead. (otherwise potential proxy settings are not used)
    * @deprecated
    */
   static getProviderFromEmail(email: string) {

node/test/test.js

@@ -769,7 +769,7 @@ describe('Integration tests', function () {
   this.beforeAll(async function () {
     if (!process.env.DCC_NEW_TMP_EMAIL) {
       console.log(
-        'Missing DCC_NEW_TMP_EMAIL environment variable!, skip intergration tests'
+        'Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests'
       )
       this.skip()
     }
@@ -777,7 +777,7 @@ describe('Integration tests', function () {
     account = await createTempUser(process.env.DCC_NEW_TMP_EMAIL)
     if (!account || !account.email || !account.password) {
       console.log(
-        "We didn't got back an account from the api, skip intergration tests"
+        "We didn't got back an account from the api, skip integration tests"
       )
       this.skip()
     }

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.105.0"
+  "version": "1.111.0"
 }

python/CHANGELOG

@@ -64,7 +64,7 @@
 - use new experimental full-Rust Delta Chat core
 - support Autocrypt Setup Messages 
 - remove synchronous events 
-- use CircleCI for continous integration and packaging of Linux wheels
+- use CircleCI for continuous integration and packaging of Linux wheels
 - use docker image for building wheels
 - fix code documentation links 
 

python/doc/conf.py

@@ -11,7 +11,8 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
+import sys
+import os
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

python/examples/echo_and_quit.py

@@ -14,10 +14,10 @@ class EchoPlugin:
             message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
-                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
+                message.chat.send_text(f"echoing system message from {addr}:\n{message}")
             else:
                 text = message.text
-                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+                message.chat.send_text(f"echoing from {addr}:\n{text}")
 
     @account_hookimpl
     def ac_message_delivered(self, message):

python/examples/group_tracking.py

@@ -14,7 +14,7 @@ class GroupTrackingPlugin:
             message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+            message.chat.send_text(f"echoing from {addr}:\n{text}")
 
     @account_hookimpl
     def ac_outgoing_message(self, message):
@@ -28,24 +28,28 @@ class GroupTrackingPlugin:
     def ac_chat_modified(self, chat):
         print("ac_chat_modified:", chat.id, chat.get_name())
         for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
+            print(f"chat member: {member.addr}")
 
     @account_hookimpl
     def ac_member_added(self, chat, contact, actor, message):
         print(
             "ac_member_added {} to chat {} from {}".format(
-                contact.addr, chat.id, actor or message.get_sender_contact().addr
-            )
+                contact.addr,
+                chat.id,
+                actor or message.get_sender_contact().addr,
+            ),
         )
         for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
+            print(f"chat member: {member.addr}")
 
     @account_hookimpl
     def ac_member_removed(self, chat, contact, actor, message):
         print(
             "ac_member_removed {} from chat {} by {}".format(
-                contact.addr, chat.id, actor or message.get_sender_contact().addr
-            )
+                contact.addr,
+                chat.id,
+                actor or message.get_sender_contact().addr,
+            ),
         )
 
 

python/examples/test_examples.py

@@ -13,8 +13,8 @@ def datadir():
         datadir = path.join("test-data")
         if datadir.isdir():
             return datadir
-    else:
-        pytest.skip("test-data directory not found")
+    pytest.skip("test-data directory not found")
+    return None
 
 
 def test_echo_quit_plugin(acfactory, lp):
@@ -47,7 +47,7 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines(
         """
         *ac_configure_completed*
-    """
+    """,
     )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
@@ -61,7 +61,7 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines(
         """
         *ac_chat_modified*bot test group*
-    """
+    """,
     )
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
@@ -76,8 +76,9 @@ def test_group_tracking_plugin(acfactory, lp):
         """
         *ac_member_added {}*from*{}*
     """.format(
-            contact3.addr, ac1.get_config("addr")
-        )
+            contact3.addr,
+            ac1.get_config("addr"),
+        ),
     )
 
     lp.sec("contact successfully added, now removing")
@@ -86,6 +87,7 @@ def test_group_tracking_plugin(acfactory, lp):
         """
         *ac_member_removed {}*from*{}*
     """.format(
-            contact3.addr, ac1.get_config("addr")
-        )
+            contact3.addr,
+            ac1.get_config("addr"),
+        ),
     )

python/pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
 dependencies = [
     "cffi>=1.0.0",
     "imap-tools",
+    "importlib_metadata;python_version<'3.8'",
     "pluggy",
     "requests",
 ]
@@ -36,13 +37,22 @@ dynamic = [
 [project.entry-points.pytest11]
 "deltachat.testplugin" = "deltachat.testplugin"
 
+[tool.setuptools.package-data]
+deltachat = [
+    "py.typed"
+]
+
 [tool.setuptools_scm]
 root = ".."
-tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
-git_describe_command = "git describe --dirty --tags --long --match py-*.*"
+tag_regex = '^(?P<prefix>v)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
+git_describe_command = "git describe --dirty --tags --long --match v*.*"
 
 [tool.black]
 line-length = 120
 
+[tool.ruff]
+select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP004", "UP010", "UP031", "UP032", "ANN204"]
+line-length = 120
+
 [tool.isort]
-profile = "black"
+profile = "black"

python/src/deltachat/__init__.py

@@ -1,6 +1,9 @@
 import sys
 
-from pkg_resources import DistributionNotFound, get_distribution
+if sys.version_info >= (3, 8):
+    from importlib.metadata import PackageNotFoundError, version
+else:
+    from importlib_metadata import PackageNotFoundError, version
 
 from . import capi, events, hookspec  # noqa
 from .account import Account, get_core_info  # noqa
@@ -11,8 +14,8 @@ from .hookspec import account_hookimpl, global_hookimpl  # noqa
 from .message import Message  # noqa
 
 try:
-    __version__ = get_distribution(__name__).version
-except DistributionNotFound:
+    __version__ = version(__name__)
+except PackageNotFoundError:
     # package is not installed
     __version__ = "0.0.0.dev0-unknown"
 
@@ -36,7 +39,8 @@ register_global_plugin(events)
 
 def run_cmdline(argv=None, account_plugins=None):
     """Run a simple default command line app, registering the specified
-    account plugins."""
+    account plugins.
+    """
     import argparse
 
     if argv is None:
@@ -54,6 +58,7 @@ def run_cmdline(argv=None, account_plugins=None):
 
     ac.run_account(addr=args.email, password=args.password, account_plugins=account_plugins, show_ffi=args.show_ffi)
 
-    print("{}: waiting for message".format(ac.get_config("addr")))
+    addr = ac.get_config("addr")
+    print(f"{addr}: waiting for message")
 
     ac.wait_shutdown()

python/src/deltachat/_build.py

@@ -102,8 +102,8 @@ def find_header(flags):
                     printf("%s", _dc_header_file_location());
                     return 0;
                 }
-            """
-                )
+            """,
+                ),
             )
         cwd = os.getcwd()
         try:
@@ -171,7 +171,7 @@ def extract_defines(flags):
             match = defines_re.match(line)
             if match:
                 defines.append(match.group(1))
-    return "\n".join("#define {} ...".format(d) for d in defines)
+    return "\n".join(f"#define {d} ..." for d in defines)
 
 
 def ffibuilder():
@@ -198,7 +198,7 @@ def ffibuilder():
         typedef int... time_t;
         void free(void *ptr);
         extern int dc_event_has_string_data(int);
-    """
+    """,
     )
     function_defs = extract_functions(flags)
     defines = extract_defines(flags)

python/src/deltachat/account.py

@@ -1,13 +1,12 @@
-""" Account class implementation. """
+"""Account class implementation."""
 
-from __future__ import print_function
 
 import os
 from array import array
 from contextlib import contextmanager
 from email.utils import parseaddr
 from threading import Event
-from typing import Any, Dict, Generator, List, Optional, Union
+from typing import Any, Dict, Generator, List, Optional, Union, TYPE_CHECKING
 
 from . import const, hookspec
 from .capi import ffi, lib
@@ -20,10 +19,12 @@ from .cutil import (
     from_optional_dc_charpointer,
     iter_array,
 )
-from .events import EventThread, FFIEventLogger
 from .message import Message
 from .tracker import ConfigureTracker, ImexTracker
 
+if TYPE_CHECKING:
+    from .events import FFIEventTracker
+
 
 class MissingCredentials(ValueError):
     """Account is missing `addr` and `mail_pw` config values."""
@@ -39,7 +40,7 @@ def get_core_info():
             ffi.gc(
                 lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
                 lib.dc_context_unref,
-            )
+            ),
         )
 
 
@@ -54,7 +55,7 @@ def get_dc_info_as_dict(dc_context):
     return info_dict
 
 
-class Account(object):
+class Account:
     """Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
@@ -62,7 +63,12 @@ class Account(object):
 
     MissingCredentials = MissingCredentials
 
+    _logid: str
+    _evtracker: "FFIEventTracker"
+
     def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None:
+        from .events import EventThread
+
         """initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -84,7 +90,7 @@ class Account(object):
 
         ptr = lib.dc_context_new_closed(db_path) if closed else lib.dc_context_new(ffi.NULL, db_path, ffi.NULL)
         if ptr == ffi.NULL:
-            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
+            raise ValueError(f"Could not dc_context_new: {os_name} {db_path}")
         self._dc_context = ffi.gc(
             ptr,
             lib.dc_context_unref,
@@ -115,8 +121,8 @@ class Account(object):
         """re-enable logging."""
         self._logging = True
 
-    def __repr__(self):
-        return "<Account path={}>".format(self.db_path)
+    def __repr__(self) -> str:
+        return f"<Account path={self.db_path}>"
 
     # def __del__(self):
     #    self.shutdown()
@@ -127,7 +133,7 @@ class Account(object):
 
     def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
-            raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(name, self._configkeys))
+            raise KeyError(f"{name!r} not a valid config key, existing keys: {self._configkeys!r}")
 
     def get_info(self) -> Dict[str, str]:
         """return dictionary of built config parameters."""
@@ -141,7 +147,7 @@ class Account(object):
         log("=============== " + self.get_config("displayname") + " ===============")
         cursor = 0
         for name, val in self.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
+            entry = f"{name.upper()}={val}"
             if cursor + len(entry) > 80:
                 log("")
                 cursor = 0
@@ -153,7 +159,7 @@ class Account(object):
         """set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
-        :param value: string to set as new transalation
+        :param value: string to set as new translation
         :returns: None
         """
         bytestring = string.encode("utf8")
@@ -172,10 +178,7 @@ class Account(object):
         namebytes = name.encode("utf8")
         if isinstance(value, (int, bool)):
             value = str(int(value))
-        if value is not None:
-            valuebytes = value.encode("utf8")
-        else:
-            valuebytes = ffi.NULL
+        valuebytes = value.encode("utf8") if value is not None else ffi.NULL
         lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
     def get_config(self, name: str) -> str:
@@ -189,7 +192,7 @@ class Account(object):
             self._check_config_key(name)
         namebytes = name.encode("utf8")
         res = lib.dc_get_config(self._dc_context, namebytes)
-        assert res != ffi.NULL, "config value not found for: {!r}".format(name)
+        assert res != ffi.NULL, f"config value not found for: {name!r}"
         return from_dc_charpointer(res)
 
     def _preconfigure_keypair(self, addr: str, public: str, secret: str) -> None:
@@ -225,9 +228,10 @@ class Account(object):
         return bool(lib.dc_is_configured(self._dc_context))
 
     def is_open(self) -> bool:
-        """Determine if account is open
+        """Determine if account is open.
 
-        :returns True if account is open."""
+        :returns True if account is open.
+        """
         return bool(lib.dc_context_is_open(self._dc_context))
 
     def set_avatar(self, img_path: Optional[str]) -> None:
@@ -280,9 +284,9 @@ class Account(object):
         :returns: :class:`deltachat.contact.Contact` instance.
         """
         (name, addr) = self.get_contact_addr_and_name(obj, name)
-        name = as_dc_charpointer(name)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
+        name_c = as_dc_charpointer(name)
+        addr_c = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, name_c, addr_c)
         return Contact(self, contact_id)
 
     def get_contact(self, obj) -> Optional[Contact]:
@@ -298,12 +302,12 @@ class Account(object):
             addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
         elif isinstance(obj, Contact):
             if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
+                raise ValueError(f"account mismatch {obj}")
             addr, displayname = obj.addr, obj.name
         elif isinstance(obj, str):
             displayname, addr = parseaddr(obj)
         else:
-            raise TypeError("don't know how to create chat for %r" % (obj,))
+            raise TypeError(f"don't know how to create chat for {obj!r}")
 
         if name is None and displayname:
             name = displayname
@@ -359,18 +363,18 @@ class Account(object):
         :returns: list of :class:`deltachat.contact.Contact` objects.
         """
         flags = 0
-        query = as_dc_charpointer(query)
+        query_c = as_dc_charpointer(query)
         if only_verified:
             flags |= const.DC_GCL_VERIFIED_ONLY
         if with_self:
             flags |= const.DC_GCL_ADD_SELF
-        dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref)
+        dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query_c), lib.dc_array_unref)
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def get_fresh_messages(self) -> Generator[Message, None, None]:
         """yield all fresh messages from all chats."""
         dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
+        return (x for x in iter_array(dc_array, lambda x: Message.from_db(self, x)) if x is not None)
 
     def create_chat(self, obj) -> Chat:
         """Create a 1:1 chat with Account, Contact or e-mail address."""
@@ -415,7 +419,7 @@ class Account(object):
     def get_device_chat(self) -> Chat:
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
-    def get_message_by_id(self, msg_id: int) -> Message:
+    def get_message_by_id(self, msg_id: int) -> Optional[Message]:
         """return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
@@ -430,7 +434,7 @@ class Account(object):
         """
         res = lib.dc_get_chat(self._dc_context, chat_id)
         if res == ffi.NULL:
-            raise ValueError("cannot get chat with id={}".format(chat_id))
+            raise ValueError(f"cannot get chat with id={chat_id}")
         lib.dc_chat_unref(res)
         return Chat(self, chat_id)
 
@@ -543,11 +547,11 @@ class Account(object):
         return from_dc_charpointer(res)
 
     def check_qr(self, qr):
-        """check qr code and return :class:`ScannedQRCode` instance representing the result"""
+        """check qr code and return :class:`ScannedQRCode` instance representing the result."""
         res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref)
         lot = DCLot(res)
         if lot.state() == const.DC_QR_ERROR:
-            raise ValueError("invalid or unknown QR code: {}".format(lot.text1()))
+            raise ValueError(f"invalid or unknown QR code: {lot.text1()}")
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
@@ -598,6 +602,8 @@ class Account(object):
     #
 
     def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
+        from .events import FFIEventLogger
+
         """get the account running, configure it if necessary. add plugins if provided.
 
         :param addr: the email address of the account
@@ -618,13 +624,11 @@ class Account(object):
             assert addr and password, "you must specify email and password once to configure this database/account"
             self.set_config("addr", addr)
             self.set_config("mail_pw", password)
-            self.set_config("mvbox_move", "0")
-            self.set_config("sentbox_watch", "0")
             self.set_config("bot", "1")
             configtracker = self.configure()
             configtracker.wait_finish()
 
-        # start IO threads and configure if neccessary
+        # start IO threads and configure if necessary
         self.start_io()
 
     def add_account_plugin(self, plugin, name=None):
@@ -662,7 +666,7 @@ class Account(object):
         return lib.dc_all_work_done(self._dc_context)
 
     def start_io(self):
-        """start this account's IO scheduling (Rust-core async scheduler)
+        """start this account's IO scheduling (Rust-core async scheduler).
 
         If this account is not configured an Exception is raised.
         You need to call account.configure() and account.wait_configure_finish()
@@ -705,12 +709,10 @@ class Account(object):
         """
         lib.dc_maybe_network(self._dc_context)
 
-    def configure(self, reconfigure: bool = False) -> ConfigureTracker:
+    def configure(self) -> ConfigureTracker:
         """Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
-
-        :param reconfigure: deprecated, doesn't need to be checked anymore.
         """
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
@@ -733,7 +735,8 @@ class Account(object):
 
     def shutdown(self) -> None:
         """shutdown and destroy account (stop callback thread, close and remove
-        underlying dc_context)."""
+        underlying dc_context).
+        """
         if self._dc_context is None:
             return
 
@@ -748,7 +751,7 @@ class Account(object):
         try:
             self._event_thread.wait(timeout=5)
         except RuntimeError as e:
-            self.log("Waiting for event thread failed: {}".format(e))
+            self.log(f"Waiting for event thread failed: {e}")
 
         if self._event_thread.is_alive():
             self.log("WARN: event thread did not terminate yet, ignoring.")
@@ -764,7 +767,7 @@ class Account(object):
 
 
 class ScannedQRCode:
-    def __init__(self, dc_lot):
+    def __init__(self, dc_lot) -> None:
         self._dc_lot = dc_lot
 
     def is_ask_verifycontact(self):

python/src/deltachat/chat.py

@@ -1,4 +1,4 @@
-""" Chat and Location related API. """
+"""Chat and Location related API."""
 
 import calendar
 import json
@@ -18,13 +18,13 @@ from .cutil import (
 from .message import Message
 
 
-class Chat(object):
+class Chat:
     """Chat object which manages members and through which you can send and retrieve messages.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id) -> None:
+    def __init__(self, account, id: int) -> None:
         from .account import Account
 
         assert isinstance(account, Account), repr(account)
@@ -37,10 +37,10 @@ class Chat(object):
         return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other) -> bool:
-        return not (self == other)
+        return not self == other
 
     def __repr__(self) -> str:
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+        return f"<Chat id={self.id} name={self.get_name()}>"
 
     @property
     def _dc_chat(self):
@@ -74,19 +74,19 @@ class Chat(object):
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
     def is_single(self) -> bool:
-        """Return True if this chat is a single/direct chat, False otherwise"""
+        """Return True if this chat is a single/direct chat, False otherwise."""
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE
 
     def is_mailinglist(self) -> bool:
-        """Return True if this chat is a mailing list, False otherwise"""
+        """Return True if this chat is a mailing list, False otherwise."""
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST
 
     def is_broadcast(self) -> bool:
-        """Return True if this chat is a broadcast list, False otherwise"""
+        """Return True if this chat is a broadcast list, False otherwise."""
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST
 
     def is_multiuser(self) -> bool:
-        """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise"""
+        """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise."""
         return lib.dc_chat_get_type(self._dc_chat) in (
             const.DC_CHAT_TYPE_GROUP,
             const.DC_CHAT_TYPE_MAILINGLIST,
@@ -94,11 +94,11 @@ class Chat(object):
         )
 
     def is_self_talk(self) -> bool:
-        """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise"""
+        """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise."""
         return bool(lib.dc_chat_is_self_talk(self._dc_chat))
 
     def is_device_talk(self) -> bool:
-        """Returns True if this chat is the "Device Messages" chat, False otherwise"""
+        """Returns True if this chat is the "Device Messages" chat, False otherwise."""
         return bool(lib.dc_chat_is_device_talk(self._dc_chat))
 
     def is_muted(self) -> bool:
@@ -109,12 +109,12 @@ class Chat(object):
         return bool(lib.dc_chat_is_muted(self._dc_chat))
 
     def is_pinned(self) -> bool:
-        """Return True if this chat is pinned, False otherwise"""
+        """Return True if this chat is pinned, False otherwise."""
         return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED
 
     def is_archived(self) -> bool:
         """Return True if this chat is archived, False otherwise.
-        :returns: True if archived, False otherwise
+        :returns: True if archived, False otherwise.
         """
         return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
 
@@ -136,7 +136,7 @@ class Chat(object):
 
     def can_send(self) -> bool:
         """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+        This is not true eg. for the contact requests or for the device-talk.
 
         :returns: True if the chat is writable, False otherwise
         """
@@ -162,12 +162,12 @@ class Chat(object):
         :param name: as a unicode string.
         :returns: True on success, False otherwise
         """
-        name = as_dc_charpointer(name)
-        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
+        name_c = as_dc_charpointer(name)
+        return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name_c))
 
     def get_color(self):
         """return the color of the chat.
-        :returns: color as 0x00rrggbb
+        :returns: color as 0x00rrggbb.
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
@@ -178,21 +178,18 @@ class Chat(object):
         return json.loads(s)
 
     def mute(self, duration: Optional[int] = None) -> None:
-        """mutes the chat
+        """mutes the chat.
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
         :returns: None
         """
-        if duration is None:
-            mute_duration = -1
-        else:
-            mute_duration = duration
+        mute_duration = -1 if duration is None else duration
         ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
         if not bool(ret):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
     def unmute(self) -> None:
-        """unmutes the chat
+        """unmutes the chat.
 
         :returns: None
         """
@@ -252,7 +249,8 @@ class Chat(object):
     def get_encryption_info(self) -> Optional[str]:
         """Return encryption info for this chat.
 
-        :returns: a string with encryption preferences of all chat members"""
+        :returns: a string with encryption preferences of all chat members
+        """
         res = lib.dc_get_chat_encrinfo(self.account._dc_context, self.id)
         return from_dc_charpointer(res)
 
@@ -284,12 +282,20 @@ class Chat(object):
         if msg.is_out_preparing():
             assert msg.id != 0
             # get a fresh copy of dc_msg, the core needs it
-            msg = Message.from_db(self.account, msg.id)
+            maybe_msg = Message.from_db(self.account, msg.id)
+            if maybe_msg is not None:
+                msg = maybe_msg
+            else:
+                raise ValueError("message does not exist")
+
         sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
+        sent_msg = Message.from_db(self.account, sent_id)
+        if sent_msg is None:
+            raise ValueError("cannot load just sent message from the database")
+        msg._dc_msg = sent_msg._dc_msg
         return msg
 
     def send_text(self, text):
@@ -446,7 +452,7 @@ class Chat(object):
         contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
-            raise ValueError("could not add contact {!r} to chat".format(contact))
+            raise ValueError(f"could not add contact {contact!r} to chat")
         return contact
 
     def remove_contact(self, obj):
@@ -459,11 +465,11 @@ class Chat(object):
         contact = self.account.get_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
-            raise ValueError("could not remove contact {!r} from chat".format(contact))
+            raise ValueError(f"could not remove contact {contact!r} from chat")
 
     def get_contacts(self):
         """get all contacts for this chat.
-        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
+        :returns: list of :class:`deltachat.contact.Contact` objects for this chat.
         """
         from .contact import Contact
 
@@ -495,7 +501,7 @@ class Chat(object):
         p = as_dc_charpointer(img_path)
         res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
         if res != 1:
-            raise ValueError("Setting Profile Image {!r} failed".format(p))
+            raise ValueError(f"Setting Profile Image {p!r} failed")
 
     def remove_profile_image(self):
         """remove group profile image.
@@ -526,13 +532,13 @@ class Chat(object):
 
     # ------  location streaming API ------------------------------
 
-    def is_sending_locations(self):
+    def is_sending_locations(self) -> bool:
         """return True if this chat has location-sending enabled currently.
         :returns: True if location sending is enabled.
         """
-        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
+        return bool(lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id))
 
-    def enable_sending_locations(self, seconds):
+    def enable_sending_locations(self, seconds) -> None:
         """enable sending locations for this chat.
 
         all subsequent messages will carry a location with them.
@@ -547,19 +553,10 @@ class Chat(object):
         :param timespan_to: a datetime object or None (indicating up till now)
         :returns: list of :class:`deltachat.chat.Location` objects.
         """
-        if timestamp_from is None:
-            time_from = 0
-        else:
-            time_from = calendar.timegm(timestamp_from.utctimetuple())
-        if timestamp_to is None:
-            time_to = 0
-        else:
-            time_to = calendar.timegm(timestamp_to.utctimetuple())
+        time_from = 0 if timestamp_from is None else calendar.timegm(timestamp_from.utctimetuple())
+        time_to = 0 if timestamp_to is None else calendar.timegm(timestamp_to.utctimetuple())
 
-        if contact is None:
-            contact_id = 0
-        else:
-            contact_id = contact.id
+        contact_id = 0 if contact is None else contact.id
 
         dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
         return [
@@ -575,7 +572,7 @@ class Chat(object):
 
 
 class Location:
-    def __init__(self, latitude, longitude, accuracy, timestamp, marker):
+    def __init__(self, latitude, longitude, accuracy, timestamp, marker) -> None:
         assert isinstance(timestamp, datetime)
         self.latitude = latitude
         self.longitude = longitude
@@ -583,5 +580,5 @@ class Location:
         self.timestamp = timestamp
         self.marker = marker
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return self.__dict__ == other.__dict__

python/src/deltachat/contact.py

@@ -1,4 +1,4 @@
-""" Contact object. """
+"""Contact object."""
 
 from datetime import date, datetime, timezone
 from typing import Optional
@@ -9,13 +9,13 @@ from .chat import Chat
 from .cutil import from_dc_charpointer, from_optional_dc_charpointer
 
 
-class Contact(object):
+class Contact:
     """Delta-Chat Contact.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
-    def __init__(self, account, id):
+    def __init__(self, account, id) -> None:
         from .account import Account
 
         assert isinstance(account, Account), repr(account)
@@ -27,11 +27,11 @@ class Contact(object):
             return False
         return self.account._dc_context == other.account._dc_context and self.id == other.id
 
-    def __ne__(self, other):
-        return not (self == other)
+    def __ne__(self, other) -> bool:
+        return not self == other
 
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
+    def __repr__(self) -> str:
+        return f"<Contact id={self.id} addr={self.addr} dc_context={self.account._dc_context}>"
 
     @property
     def _dc_contact(self):
@@ -76,7 +76,7 @@ class Contact(object):
         return lib.dc_contact_is_verified(self._dc_contact)
 
     def get_verifier(self, contact):
-        """Return the address of the contact that verified the contact"""
+        """Return the address of the contact that verified the contact."""
         return from_dc_charpointer(lib.dc_contact_get_verifier_addr(contact._dc_contact))
 
     def get_profile_image(self) -> Optional[str]:

python/src/deltachat/direct_imap.py

@@ -79,15 +79,17 @@ class DirectImap:
 
     def select_config_folder(self, config_name: str):
         """Return info about selected folder if it is
-        configured, otherwise None."""
+        configured, otherwise None.
+        """
         if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
+            config_name = f"configured_{config_name}_folder"
         foldername = self.account.get_config(config_name)
         if foldername:
             return self.select_folder(foldername)
+        return None
 
     def list_folders(self) -> List[str]:
-        """return list of all existing folder names"""
+        """return list of all existing folder names."""
         assert not self._idling
         return [folder.name for folder in self.conn.folder.list()]
 
@@ -103,7 +105,7 @@ class DirectImap:
 
     def get_all_messages(self) -> List[MailMessage]:
         assert not self._idling
-        return [mail for mail in self.conn.fetch()]
+        return list(self.conn.fetch())
 
     def get_unread_messages(self) -> List[str]:
         assert not self._idling
@@ -189,7 +191,7 @@ class DirectImap:
 
 
 class IdleManager:
-    def __init__(self, direct_imap):
+    def __init__(self, direct_imap) -> None:
         self.direct_imap = direct_imap
         self.log = direct_imap.account.log
         # fetch latest messages before starting idle so that it only
@@ -201,18 +203,18 @@ class IdleManager:
         """(blocking) wait for next idle message from server."""
         self.log("imap-direct: calling idle_check")
         res = self.direct_imap.conn.idle.poll(timeout=timeout)
-        self.log("imap-direct: idle_check returned {!r}".format(res))
+        self.log(f"imap-direct: idle_check returned {res!r}")
         return res
 
     def wait_for_new_message(self, timeout=None) -> bytes:
-        while 1:
+        while True:
             for item in self.check(timeout=timeout):
                 if b"EXISTS" in item or b"RECENT" in item:
                     return item
 
     def wait_for_seen(self, timeout=None) -> int:
         """Return first message with SEEN flag from a running idle-stream."""
-        while 1:
+        while True:
             for item in self.check(timeout=timeout):
                 if FETCH in item:
                     self.log(str(item))
@@ -221,5 +223,4 @@ class IdleManager:
 
     def done(self):
         """send idle-done to server if we are currently in idle mode."""
-        res = self.direct_imap.conn.idle.stop()
-        return res
+        return self.direct_imap.conn.idle.stop()

python/src/deltachat/events.py

@@ -13,6 +13,7 @@ from .capi import ffi, lib
 from .cutil import from_optional_dc_charpointer
 from .hookspec import account_hookimpl
 from .message import map_system_message
+from .account import Account
 
 
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
@@ -24,12 +25,18 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
 
 
 class FFIEvent:
-    def __init__(self, name: str, data1, data2):
+    def __init__(self, name: str, data1, data2) -> None:
         self.name = name
         self.data1 = data1
         self.data2 = data2
 
-    def __str__(self):
+    def __str__(self) -> str:
+        if self.name == "DC_EVENT_INFO":
+            return f"INFO {self.data2}"
+        if self.name == "DC_EVENT_WARNING":
+            return f"WARNING {self.data2}"
+        if self.name == "DC_EVENT_ERROR":
+            return f"ERROR {self.data2}"
         return "{name} data1={data1} data2={data2}".format(**self.__dict__)
 
 
@@ -62,7 +69,7 @@ class FFIEventLogger:
         locname = tname
         if self.logid:
             locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        s = f"{elapsed:2.2f} [{locname}] {message}"
 
         if os.name == "posix":
             WARN = "\033[93m"
@@ -77,7 +84,10 @@ class FFIEventLogger:
 
 
 class FFIEventTracker:
-    def __init__(self, account, timeout=None):
+    account: Account
+    _event_queue: Queue
+
+    def __init__(self, account: Account, timeout=None) -> None:
         self.account = account
         self._timeout = timeout
         self._event_queue = Queue()
@@ -97,29 +107,29 @@ class FFIEventTracker:
         timeout = timeout if timeout is not None else self._timeout
         ev = self._event_queue.get(timeout=timeout)
         if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError("unexpected event: {}".format(ev))
+            raise ValueError(f"unexpected event: {ev}")
         return ev
 
     def iter_events(self, timeout=None, check_error=True):
-        while 1:
+        while True:
             yield self.get(timeout=timeout, check_error=check_error)
 
     def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        rex = re.compile("^(?:{})$".format(event_name_regex))
+        rex = re.compile(f"^(?:{event_name_regex})$")
         for ev in self.iter_events(timeout=timeout, check_error=check_error):
             if rex.match(ev.name):
                 return ev
 
     def get_info_contains(self, regex: str) -> FFIEvent:
         rex = re.compile(regex)
-        while 1:
+        while True:
             ev = self.get_matching("DC_EVENT_INFO")
             if rex.search(ev.data2):
                 return ev
 
     def get_info_regex_groups(self, regex, check_error=True):
         rex = re.compile(regex)
-        while 1:
+        while True:
             ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
             m = rex.match(ev.data2)
             if m is not None:
@@ -128,46 +138,48 @@ class FFIEventTracker:
     def wait_for_connectivity(self, connectivity):
         """Wait for the specified connectivity.
         This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
+        again too quickly, otherwise we might miss it.
+        """
+        while True:
             if self.account.get_connectivity() == connectivity:
                 return
             self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
 
     def wait_for_connectivity_change(self, previous, expected_next):
         """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
+        Fails the test if it changes to something else.
+        """
+        while True:
             current = self.account.get_connectivity()
             if current == expected_next:
                 return
-            elif current != previous:
+            if current != previous:
                 raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
 
             self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
 
     def wait_for_all_work_done(self):
-        while 1:
+        while True:
             if self.account.all_work_done():
                 return
             self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
 
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
+        rex = re.compile(f"(?:{event_name_regex}).*")
+        while True:
             try:
                 ev = self._event_queue.get(False)
             except Empty:
                 break
             else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
+                assert not rex.match(ev.name), f"event found {ev}"
 
     def wait_securejoin_inviter_progress(self, target):
-        while 1:
+        while True:
             event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
             if event.data2 >= target:
-                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
+                print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)
                 break
 
     def wait_idle_inbox_ready(self):
@@ -176,7 +188,8 @@ class FFIEventTracker:
         - ac1 and ac2 are created
         - ac1 sends a message to ac2
         - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
-        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        - therefore no DC_EVENT_INCOMING_MSG is sent
+        """
         self.get_info_contains("INBOX: Idle entering")
 
     def wait_next_incoming_message(self):
@@ -186,14 +199,15 @@ class FFIEventTracker:
 
     def wait_next_messages_changed(self):
         """wait for and return next message-changed message or None
-        if the event contains no msgid"""
+        if the event contains no msgid
+        """
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
         return None
 
     def wait_next_reactions_changed(self):
-        """wait for and return next reactions-changed message"""
+        """wait for and return next reactions-changed message."""
         ev = self.get_matching("DC_EVENT_REACTIONS_CHANGED")
         assert ev.data1 > 0
         return self.account.get_message_by_id(ev.data2)
@@ -211,7 +225,7 @@ class EventThread(threading.Thread):
     With each Account init this callback thread is started.
     """
 
-    def __init__(self, account) -> None:
+    def __init__(self, account: Account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.daemon = True
@@ -237,36 +251,37 @@ class EventThread(threading.Thread):
     def run(self) -> None:
         """get and run events until shutdown."""
         with self.log_execution("EVENT THREAD"):
-            self._inner_run()
+            event_emitter = ffi.gc(
+                lib.dc_get_event_emitter(self.account._dc_context),
+                lib.dc_event_emitter_unref,
+            )
+            while not self._marked_for_shutdown:
+                with self.swallow_and_log_exception("Unexpected error in event thread"):
+                    event = lib.dc_get_next_event(event_emitter)
+                    if event == ffi.NULL or self._marked_for_shutdown:
+                        break
+                    self._process_event(event)
 
-    def _inner_run(self):
-        event_emitter = ffi.gc(
-            lib.dc_get_event_emitter(self.account._dc_context),
-            lib.dc_event_emitter_unref,
-        )
-        while not self._marked_for_shutdown:
-            event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL or self._marked_for_shutdown:
-                break
-            evt = lib.dc_event_get_id(event)
-            data1 = lib.dc_event_get_data1_int(event)
-            # the following code relates to the deltachat/_build.py's helper
-            # function which provides us signature info of an event call
-            evt_name = get_dc_event_name(evt)
-            if lib.dc_event_has_string_data(evt):
-                data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
-            else:
-                data2 = lib.dc_event_get_data2_int(event)
+    def _process_event(self, event) -> None:
+        evt = lib.dc_event_get_id(event)
+        data1 = lib.dc_event_get_data1_int(event)
+        # the following code relates to the deltachat/_build.py's helper
+        # function which provides us signature info of an event call
+        evt_name = get_dc_event_name(evt)
+        if lib.dc_event_has_string_data(evt):
+            data2 = from_optional_dc_charpointer(lib.dc_event_get_data2_str(event))
+        else:
+            data2 = lib.dc_event_get_data2_int(event)
 
-            lib.dc_event_unref(event)
-            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            with self.swallow_and_log_exception("ac_process_ffi_event {}".format(ffi_event)):
-                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-            for name, kwargs in self._map_ffi_event(ffi_event):
-                hook = getattr(self.account._pm.hook, name)
-                info = "call {} kwargs={} failed".format(name, kwargs)
-                with self.swallow_and_log_exception(info):
-                    hook(**kwargs)
+        lib.dc_event_unref(event)
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        with self.swallow_and_log_exception(f"ac_process_ffi_event {ffi_event}"):
+            self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
+        for name, kwargs in self._map_ffi_event(ffi_event):
+            hook = getattr(self.account._pm.hook, name)
+            info = f"call {name} kwargs={kwargs} failed"
+            with self.swallow_and_log_exception(info):
+                hook(**kwargs)
 
     @contextmanager
     def swallow_and_log_exception(self, info):
@@ -275,7 +290,7 @@ class EventThread(threading.Thread):
         except Exception as ex:
             logfile = io.StringIO()
             traceback.print_exception(*sys.exc_info(), file=logfile)
-            self.account.log("{}\nException {}\nTraceback:\n{}".format(info, ex, logfile.getvalue()))
+            self.account.log(f"{info}\nException {ex}\nTraceback:\n{logfile.getvalue()}")
 
     def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
@@ -285,30 +300,32 @@ class EventThread(threading.Thread):
             if data1 == 0 or data1 == 1000:
                 success = data1 == 1000
                 comment = ffi_event.data2
-                yield "ac_configure_completed", dict(success=success, comment=comment)
+                yield "ac_configure_completed", {"success": success, "comment": comment}
         elif name == "DC_EVENT_INCOMING_MSG":
             msg = account.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+            if msg is not None:
+                yield map_system_message(msg) or ("ac_incoming_message", {"message": msg})
         elif name == "DC_EVENT_MSGS_CHANGED":
             if ffi_event.data2 != 0:
                 msg = account.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or (
-                        "ac_incoming_message",
-                        dict(message=msg),
-                    )
+                if msg is not None:
+                    if msg.is_outgoing():
+                        res = map_system_message(msg)
+                        if res and res[0].startswith("ac_member"):
+                            yield res
+                        yield "ac_outgoing_message", {"message": msg}
+                    elif msg.is_in_fresh():
+                        yield map_system_message(msg) or (
+                            "ac_incoming_message",
+                            {"message": msg},
+                        )
         elif name == "DC_EVENT_REACTIONS_CHANGED":
             assert ffi_event.data1 > 0
             msg = account.get_message_by_id(ffi_event.data2)
-            yield "ac_reactions_changed", dict(message=msg)
+            yield "ac_reactions_changed", {"message": msg}
         elif name == "DC_EVENT_MSG_DELIVERED":
             msg = account.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
+            yield "ac_message_delivered", {"message": msg}
         elif name == "DC_EVENT_CHAT_MODIFIED":
             chat = account.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)
+            yield "ac_chat_modified", {"chat": chat}

python/src/deltachat/hookspec.py

@@ -1,4 +1,4 @@
-""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
+"""Hooks for Python bindings to Delta Chat Core Rust CFFI."""
 
 import pluggy
 

python/src/deltachat/message.py

@@ -1,4 +1,4 @@
-""" The Message object. """
+"""The Message object."""
 
 import json
 import os
@@ -12,14 +12,14 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_char
 from .reactions import Reactions
 
 
-class Message(object):
+class Message:
     """Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
     :class:`deltachat.chat.Chat`.
     """
 
-    def __init__(self, account, dc_msg):
+    def __init__(self, account, dc_msg) -> None:
         self.account = account
         assert isinstance(self.account._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
@@ -33,24 +33,24 @@ class Message(object):
             return False
         return self.account == other.account and self.id == other.id
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         c = self.get_sender_contact()
         typ = "outgoing" if self.is_outgoing() else "incoming"
-        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ,
-            self.is_system_message(),
-            repr(self.text[:10]),
-            self.id,
-            c.id,
-            c.addr,
-            self.chat.id,
-            self.chat.get_name(),
+        return (
+            f"<Message {typ} sys={self.is_system_message()} {repr(self.text[:100])} "
+            f"id={self.id} sender={c.id}/{c.addr} chat={self.chat.id}/{self.chat.get_name()}>"
         )
 
     @classmethod
-    def from_db(cls, account, id):
+    def from_db(cls, account, id) -> Optional["Message"]:
+        """Attempt to load the message from the database given its ID.
+
+        None is returned if the message does not exist, i.e. deleted."""
         assert id > 0
-        return cls(account, ffi.gc(lib.dc_get_msg(account._dc_context, id), lib.dc_msg_unref))
+        res = lib.dc_get_msg(account._dc_context, id)
+        if res == ffi.NULL:
+            return None
+        return cls(account, ffi.gc(res, lib.dc_msg_unref))
 
     @classmethod
     def new_empty(cls, account, view_type):
@@ -59,10 +59,7 @@ class Message(object):
         :param view_type: the message type code or one of the strings:
            "text", "audio", "video", "file", "sticker", "videochat", "webxdc"
         """
-        if isinstance(view_type, int):
-            view_type_code = view_type
-        else:
-            view_type_code = get_viewtype_code_from_name(view_type)
+        view_type_code = view_type if isinstance(view_type, int) else get_viewtype_code_from_name(view_type)
         return Message(
             account,
             ffi.gc(lib.dc_msg_new(account._dc_context, view_type_code), lib.dc_msg_unref),
@@ -118,7 +115,7 @@ class Message(object):
         """set file for this message from path and mime_type."""
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
-            raise ValueError("path does not exist: {!r}".format(path))
+            raise ValueError(f"path does not exist: {path!r}")
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
@@ -129,7 +126,7 @@ class Message(object):
 
     @props.with_doc
     def filemime(self) -> str:
-        """mime type of the file (if it exists)"""
+        """mime type of the file (if it exists)."""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
     def get_status_updates(self, serial: int = 0) -> list:
@@ -141,7 +138,7 @@ class Message(object):
         :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates.
         """
         return json.loads(
-            from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial))
+            from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial)),
         )
 
     def send_status_update(self, json_data: Union[str, dict], description: str) -> bool:
@@ -158,8 +155,11 @@ class Message(object):
             json_data = json.dumps(json_data, default=str)
         return bool(
             lib.dc_send_webxdc_status_update(
-                self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description)
-            )
+                self.account._dc_context,
+                self.id,
+                as_dc_charpointer(json_data),
+                as_dc_charpointer(description),
+            ),
         )
 
     def send_reaction(self, reaction: str):
@@ -232,16 +232,18 @@ class Message(object):
         ts = lib.dc_msg_get_received_timestamp(self._dc_msg)
         if ts:
             return datetime.fromtimestamp(ts, timezone.utc)
+        return None
 
     @props.with_doc
     def ephemeral_timer(self):
-        """Ephemeral timer in seconds
+        """Ephemeral timer in seconds.
 
         :returns: timer in seconds or None if there is no timer
         """
         timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
         if timer:
             return timer
+        return None
 
     @props.with_doc
     def ephemeral_timestamp(self):
@@ -255,23 +257,25 @@ class Message(object):
 
     @property
     def quoted_text(self) -> Optional[str]:
-        """Text inside the quote
+        """Text inside the quote.
 
-        :returns: Quoted text"""
+        :returns: Quoted text
+        """
         return from_optional_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
 
     @property
     def quote(self):
-        """Quote getter
+        """Quote getter.
 
-        :returns: Quoted message, if found in the database"""
+        :returns: Quoted message, if found in the database
+        """
         msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
         if msg:
             return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
 
     @quote.setter
     def quote(self, quoted_message):
-        """Quote setter"""
+        """Quote setter."""
         lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
 
     def force_plaintext(self) -> None:
@@ -286,7 +290,7 @@ class Message(object):
 
         :returns: email-mime message object (with headers only, no body).
         """
-        import email.parser
+        import email
 
         mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
         if mime_headers:
@@ -297,7 +301,7 @@ class Message(object):
 
     @property
     def error(self) -> Optional[str]:
-        """Error message"""
+        """Error message."""
         return from_optional_dc_charpointer(lib.dc_msg_get_error(self._dc_msg))
 
     @property
@@ -493,7 +497,8 @@ def get_viewtype_code_from_name(view_type_name):
     if code is not None:
         return code
     raise ValueError(
-        "message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.keys()))
+        "message typecode not found for {!r}, "
+        "available {!r}".format(view_type_name, list(_view_type_mapping.keys())),
     )
 
 
@@ -506,14 +511,11 @@ def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
         if not res:
-            return
+            return None
         action, affected, actor = res
         affected = msg.account.get_contact_by_addr(affected)
-        if actor == "me":
-            actor = None
-        else:
-            actor = msg.account.get_contact_by_addr(actor)
-        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        actor = None if actor == "me" else msg.account.get_contact_by_addr(actor)
+        d = {"chat": msg.chat, "contact": affected, "actor": actor, "message": msg}
         return "ac_member_" + res[0], d
 
 
@@ -528,8 +530,8 @@ def extract_addr(text):
 def parse_system_add_remove(text):
     """return add/remove info from parsing the given system message text.
 
-    returns a (action, affected, actor) triple"""
-
+    returns a (action, affected, actor) triple
+    """
     # You removed member a@b.
     # You added member a@b.
     # Member Me (x@y) removed by a@b.