Wrap IMAP streams into BufStream

BufWriter buffers only the writer side, but not the reader side.
With BufStream we move all the buffering to the core
and do not expect any buffering to happen inside the dependencies.

.cargo/config.toml

@@ -0,0 +1,11 @@
+[env]
+# In unoptimised builds tokio tends to use a lot of stack space when
+# creating some complicated futures, tokio has an open issue for this:
+# https://github.com/tokio-rs/tokio/issues/2055.  Some of our tests
+# manage to not fit in the default 2MiB stack anymore due to this, so
+# while the issue is not resolved we want to work around this.
+# Because compiling optimised builds takes a very long time we prefer
+# to avoid that.  Setting this environment variable ensures that when
+# invoking `cargo test` threads are allowed to have a large enough
+# stack size without needing to use an optimised build.
+RUST_MIN_STACK = "8388608"

.gitattributes

@@ -4,7 +4,7 @@
 
 # This directory contains email messages verbatim, and changing CRLF to
 # LF will corrupt them.
-test-data/* text=false
+test-data/** text=false
 
 # binary files should be detected by git, however, to be sure, you can add them here explicitly
 *.png  binary

.github/dependabot.yml

@@ -3,7 +3,7 @@ updates:
   - package-ecosystem: "cargo"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "monthly"
     commit-message:
       prefix: "cargo"
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 50

.github/mergeable.yml

@@ -5,22 +5,22 @@ mergeable:
     validate:
       - do: or
         validate:
-        - do: description
-          must_include:
-            regex: '#skip-changelog'
-        - do: and
-          validate:
-            - do: dependent
-              changed:
-                file: 'src/**'
-                required: ['CHANGELOG.md']
-            - do: dependent
-              changed:
-                file: 'deltachat-ffi/**'
-                required: ['CHANGELOG.md']
+          - do: description
+            must_include:
+              regex: "#skip-changelog"
+          - do: and
+            validate:
+              - do: dependent
+                changed:
+                  file: "src/**"
+                  required: ["CHANGELOG.md"]
+              - do: dependent
+                changed:
+                  file: "deltachat-ffi/src/**"
+                  required: ["CHANGELOG.md"]
     fail:
       - do: checks
-        status: 'action_required' 
+        status: "action_required"
         payload:
-          title: Changlog might need an update
+          title: Changelog might need an update
           summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -1,51 +1,55 @@
 name: Rust CI
 
+# Cancel previously started workflow runs
+# when the branch is updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   pull_request:
   push:
     branches:
       - master
-      - staging
-      - trying
 
 env:
   RUSTFLAGS: -Dwarnings
 
 jobs:
+  lint:
+    name: Rustfmt and Clippy
+    runs-on: ubuntu-latest
+    env:
+      RUSTUP_TOOLCHAIN: 1.68.0
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install rustfmt and clippy
+        run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v2
+      - name: Run rustfmt
+        run: cargo fmt --all -- --check
+      - name: Run clippy
+        run: scripts/clippy.sh
 
-  fmt:
-    name: Rustfmt
+  cargo_deny:
+    name: cargo deny
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v3
+      - uses: EmbarkStudios/cargo-deny-action@v1
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
+          arguments: --all-features --workspace
+          command: check
+          command-arguments: "-Dwarnings"
 
-  run_clippy:
+  provider_database:
+    name: Check provider database
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: stable
-            components: clippy
-            override: true
-      - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples
+      - uses: actions/checkout@v3
+      - name: Check provider database
+        run: scripts/update-provider-database.sh
 
   docs:
     name: Rust doc comments
@@ -54,90 +58,103 @@ jobs:
       RUSTDOCFLAGS: -Dwarnings
     steps:
       - name: Checkout sources
-        uses: actions/checkout@v2
-      - name: Install rust stable toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          components: rust-docs
-          override: true
+        uses: actions/checkout@v3
       - name: Cache rust cargo artifacts
-        uses: swatinem/rust-cache@v1
+        uses: swatinem/rust-cache@v2
       - name: Rustdoc
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --document-private-items --no-deps
+        run: cargo doc --document-private-items --no-deps
 
   build_and_test:
     name: Build and test
     strategy:
+      fail-fast: false
       matrix:
         include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
+          # Currently used Rust version.
           - os: ubuntu-latest
-            rust: 1.54.0
+            rust: 1.68.0
             python: 3.9
           - os: windows-latest
-            rust: 1.54.0
+            rust: 1.68.0
             python: false # Python bindings compilation on Windows is not supported.
 
-          # Minimum Supported Rust Version = 1.51.0
+          # Minimum Supported Rust Version = 1.64.0
           #
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
           # built.
           - os: ubuntu-latest
-            rust: 1.51.0
+            rust: 1.64.0
             python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@master
+      - uses: actions/checkout@master
 
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
+      - name: Install Rust ${{ matrix.rust }}
+        run: rustup toolchain install --profile minimal ${{ matrix.rust }}
+      - run: rustup override set ${{ matrix.rust }}
 
-    - name: Cache rust cargo artifacts
-      uses: swatinem/rust-cache@v1
+      - name: Cache rust cargo artifacts
+        uses: swatinem/rust-cache@v2
 
-    - name: check
-      uses: actions-rs/cargo@v1
-      with:
-        command: check
-        args: --all --bins --examples --tests --features repl --benches
+      - name: Check
+        run: cargo check --workspace --bins --examples --tests --benches
 
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --all
+      - name: Tests
+        run: cargo test --workspace
 
-    - name: install python
-      if: ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
+      - name: Test cargo vendor
+        run: cargo vendor
 
-    - name: install tox
-      if: ${{ matrix.python }}
-      run: pip install tox
+      - name: Install python
+        if: ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
 
-    - name: build C library
-      if: ${{ matrix.python }}
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: -p deltachat_ffi
+      - name: Install tox
+        if: ${{ matrix.python }}
+        run: pip install tox
 
-    - name: run python tests
-      if: ${{ matrix.python }}
-      env:
-        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-        DCC_RS_TARGET: debug
-        DCC_RS_DEV: ${{ github.workspace }}
-      working-directory: python
-      run: tox -e lint,mypy,doc,py3
+      - name: Build C library
+        if: ${{ matrix.python }}
+        run: cargo build -p deltachat_ffi --features jsonrpc
+
+      - name: Run python tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          DCC_RS_TARGET: debug
+          DCC_RS_DEV: ${{ github.workspace }}
+        working-directory: python
+        run: tox -e lint,mypy,doc,py3
+
+      - name: Build deltachat-rpc-server
+        if: ${{ matrix.python }}
+        run: cargo build -p deltachat-rpc-server
+
+      - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.python }}
+        run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
+
+      - name: Run deltachat-rpc-client tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+        working-directory: deltachat-rpc-client
+        run: tox -e py3,lint
+
+      - name: Install pypy
+        if: ${{ matrix.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "pypy${{ matrix.python }}"
+
+      - name: Run pypy tests
+        if: ${{ matrix.python }}
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          DCC_RS_TARGET: debug
+          DCC_RS_DEV: ${{ github.workspace }}
+        working-directory: python
+        run: tox -e pypy3

.github/workflows/deltachat-rpc-server.yml

@@ -0,0 +1,136 @@
+# Manually triggered action to build deltachat-rpc-server binaries.
+
+name: Build deltachat-rpc-server binaries
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+
+jobs:
+  # Build a version statically linked against musl libc
+  # to avoid problems with glibc version incompatibility.
+  build_static_linux:
+    name: Build deltachat-rpc-server for Linux
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup rust target
+        run: rustup target add x86_64-unknown-linux-musl
+
+      - name: Install musl-gcc
+        run: sudo apt install musl-tools
+
+      - name: Build
+        env:
+          RUSTFLAGS: "-C link-arg=-s"
+        run: cargo build --release --target x86_64-unknown-linux-musl -p deltachat-rpc-server --features vendored
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-x86_64
+          path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server
+          if-no-files-found: error
+
+  build_linux:
+    name: Cross-compile deltachat-rpc-server for aarch64 and armv7 Linux
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build
+        run: sh scripts/zig-rpc-server.sh
+
+      - name: Upload aarch64 binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-aarch64
+          path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload armv7 binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-armv7
+          path: target/armv7-unknown-linux-musleabihf/release/deltachat-rpc-server
+          if-no-files-found: error
+
+  build_android:
+    name: Cross-compile deltachat-rpc-server for Android (armeabi-v7a, arm64-v8a, x86 and x86_64)
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: nttld/setup-ndk@v1
+        id: setup-ndk
+        with:
+          ndk-version: r21d
+
+      - name: Build
+        env:
+          ANDROID_NDK_ROOT: ${{ steps.setup-ndk.outputs.ndk-path }}
+        run: sh scripts/android-rpc-server.sh
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-android-armv7
+          path: target/armv7-linux-androideabi/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-android-aarch64
+          path: target/aarch64-linux-android/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-android-i686
+          path: target/i686-linux-android/release/deltachat-rpc-server
+          if-no-files-found: error
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-android-x86_64
+          path: target/x86_64-linux-android/release/deltachat-rpc-server
+          if-no-files-found: error
+
+  build_windows:
+    name: Build deltachat-rpc-server for Windows
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            artifact: win32.exe
+            path: deltachat-rpc-server.exe
+            target: i686-pc-windows-msvc
+
+          - os: windows-latest
+            artifact: win64.exe
+            path: deltachat-rpc-server.exe
+            target: x86_64-pc-windows-msvc
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup rust target
+        run: rustup target add ${{ matrix.target }}
+
+      - name: Build
+        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-rpc-server-${{ matrix.artifact }}
+          path: target/${{ matrix.target}}/release/${{ matrix.path }}
+          if-no-files-found: error

.github/workflows/dependabot.yml

@@ -0,0 +1,21 @@
+name: Dependabot auto-approve
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  dependabot:
+    runs-on: ubuntu-latest
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - name: Dependabot metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.1.1
+        with:
+          github-token: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/jsonrpc-client-npm-package.yml

@@ -0,0 +1,82 @@
+name: "jsonrpc js client build"
+on:
+  pull_request:
+  push:
+    tags:
+      - "*"
+      - "!py-*"
+
+jobs:
+  pack-module:
+    name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat"
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Install tree
+        run: sudo apt install tree
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+        continue-on-error: true
+      - name: Get Pull Request ID
+        id: prepare
+        run: |
+          tag=${{ steps.tag.outputs.tag }}
+          if [ -z "$tag" ]; then
+            node -e "console.log('DELTACHAT_JSONRPC_TAR_GZ=deltachat-jsonrpc-client-' + '${{ github.ref }}'.split('/')[2] + '.tar.gz')" >> $GITHUB_ENV
+          else
+            echo "DELTACHAT_JSONRPC_TAR_GZ=deltachat-jsonrpc-client-${{ steps.tag.outputs.tag }}.tar.gz" >> $GITHUB_ENV
+            echo "No preview will be uploaded this time, but the $tag release"
+          fi
+      - name: System info
+        run: |
+          npm --version
+          node --version
+          echo $DELTACHAT_JSONRPC_TAR_GZ
+      - name: Install dependencies without running scripts
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm install --ignore-scripts
+      - name: Package
+        shell: bash
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run build
+          npm pack .
+          ls -lah
+          mv $(find deltachat-jsonrpc-client-*) $DELTACHAT_JSONRPC_TAR_GZ
+      - name: Upload Prebuild
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-jsonrpc-client.tgz
+          path: deltachat-jsonrpc/typescript/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
+      # Upload to download.delta.chat/node/preview/
+      - name: Upload deltachat-jsonrpc-client preview to download.delta.chat/node/preview/
+        if: ${{ ! steps.tag.outputs.tag }}
+        id: upload-preview
+        shell: bash
+        run: |
+          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
+          chmod 600 __TEMP_INPUT_KEY_FILE
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
+        continue-on-error: true
+      - name: Post links to details
+        if: steps.upload-preview.outcome == 'success'
+        run: node ./node/scripts/postLinksToDetails.js
+        env:
+          URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
+      # Upload to download.delta.chat/node/
+      - name: Upload deltachat-jsonrpc-client build to download.delta.chat/node/
+        if: ${{ steps.tag.outputs.tag }}
+        id: upload
+        shell: bash
+        run: |
+          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
+          chmod 600 __TEMP_INPUT_KEY_FILE
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"

.github/workflows/jsonrpc.yml

@@ -0,0 +1,45 @@
+name: JSON-RPC API Test
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_MIN_STACK: "8388608"
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+      - name: Add Rust cache
+        uses: Swatinem/rust-cache@v2
+      - name: npm install
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm install
+      - name: Build TypeScript, run Rust tests, generate bindings
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run build
+      - name: Run integration tests
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run test
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+      - name: make sure websocket server version still builds
+        run: |
+          cd deltachat-jsonrpc
+          cargo build --bin deltachat-jsonrpc-server --features webserver
+      - name: Run linter
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run prettier:check

.github/workflows/node-delete-preview.yml

@@ -0,0 +1,31 @@
+# documentation: https://github.com/deltachat/sysadmin/tree/master/download.delta.chat
+name: Delete node PR previews
+
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  delete:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get Pull Request ID
+        id: getid
+        run: |
+          export PULLREQUEST_ID=$(jq .number < $GITHUB_EVENT_PATH)
+          echo "prid=$PULLREQUEST_ID" >> $GITHUB_OUTPUT
+      - name: Renaming
+        run: |
+          # create empty file to copy it over the outdated deliverable on download.delta.chat
+          echo "This preview build is outdated and has been removed." > empty
+          cp empty deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz
+      - name: Replace builds with dummy files
+        uses: horochx/deploy-via-scp@v1.0.1
+        with:
+          user: ${{ secrets.USERNAME }}
+          key: ${{ secrets.SSH_KEY }}
+          host: "download.delta.chat"
+          port: 22
+          local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
+          remote: "/var/www/html/download/node/preview/"

.github/workflows/node-docs.yml

@@ -0,0 +1,34 @@
+name: Generate & upload node.js documentation
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16.x
+
+      - name: npm install and generate documentation
+        run: |
+          cd node
+          npm i --ignore-scripts
+          npx typedoc
+          mv docs js
+
+      - name: Upload
+        uses: horochx/deploy-via-scp@v1.0.1
+        with:
+          user: ${{ secrets.USERNAME }}
+          key: ${{ secrets.KEY }}
+          host: "delta.chat"
+          port: 22
+          local: "node/js"
+          remote: "/var/www/html/"

.github/workflows/node-package.yml

@@ -0,0 +1,164 @@
+name: "node.js build"
+on:
+  pull_request:
+  push:
+    tags:
+      - "*"
+      - "!py-*"
+
+jobs:
+  prebuild:
+    name: Prebuild
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+      - name: System info
+        run: |
+          rustc -vV
+          rustup -vV
+          cargo -vV
+          npm --version
+          node --version
+
+      - name: Cache node modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ env.APPDATA }}/npm-cache
+            ~/.npm
+          key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry/
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2
+
+      - name: Install dependencies & build
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cd node
+          npm install --verbose
+
+      - name: Build Prebuild
+        run: |
+          cd node
+          npm run prebuildify
+          tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds .
+
+      - name: Upload Prebuild
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.os }}
+          path: node/${{ matrix.os }}.tar.gz
+
+  pack-module:
+    needs: prebuild
+    name: Package deltachat-node and upload to download.delta.chat
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Install tree
+        run: sudo apt install tree
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+        continue-on-error: true
+      - name: Get Pull Request ID
+        id: prepare
+        run: |
+          tag=${{ steps.tag.outputs.tag }}
+          if [ -z "$tag" ]; then
+            node -e "console.log('DELTACHAT_NODE_TAR_GZ=deltachat-node-' + '${{ github.ref }}'.split('/')[2] + '.tar.gz')" >> $GITHUB_ENV
+          else
+            echo "DELTACHAT_NODE_TAR_GZ=deltachat-node-${{ steps.tag.outputs.tag }}.tar.gz" >> $GITHUB_ENV
+            echo "No preview will be uploaded this time, but the $tag release"
+          fi
+      - name: System info
+        run: |
+          rustc -vV
+          rustup -vV
+          cargo -vV
+          npm --version
+          node --version
+          echo $DELTACHAT_NODE_TAR_GZ
+      - name: Download Ubuntu prebuild
+        uses: actions/download-artifact@v1
+        with:
+          name: ubuntu-20.04
+      - name: Download macOS prebuild
+        uses: actions/download-artifact@v1
+        with:
+          name: macos-latest
+      - name: Download Windows prebuild
+        uses: actions/download-artifact@v1
+        with:
+          name: windows-latest
+      - shell: bash
+        run: |
+          mkdir node/prebuilds
+          tar -xvzf ubuntu-20.04/ubuntu-20.04.tar.gz -C node/prebuilds
+          tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
+          tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
+          tree node/prebuilds
+          rm -rf ubuntu-20.04 macos-latest windows-latest
+      - name: Install dependencies without running scripts
+        run: |
+          npm install --ignore-scripts
+      - name: Build constants
+        run: |
+          npm run build:core:constants
+      - name: Build TypeScript part
+        run: |
+          npm run build:bindings:ts
+      - name: Package
+        shell: bash
+        run: |
+          mv node/README.md README.md
+          npm pack .
+          ls -lah
+          mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
+      - name: Upload prebuild
+        uses: actions/upload-artifact@v3
+        with:
+          name: deltachat-node.tgz
+          path: ${{ env.DELTACHAT_NODE_TAR_GZ }}
+      # Upload to download.delta.chat/node/preview/
+      - name: Upload deltachat-node preview to download.delta.chat/node/preview/
+        if: ${{ ! steps.tag.outputs.tag }}
+        id: upload-preview
+        shell: bash
+        run: |
+          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
+          chmod 600 __TEMP_INPUT_KEY_FILE
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
+        continue-on-error: true
+      - name: Post links to details
+        if: steps.upload-preview.outcome == 'success'
+        run: node ./node/scripts/postLinksToDetails.js
+        env:
+          URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # Upload to download.delta.chat/node/
+      - name: Upload deltachat-node build to download.delta.chat/node/
+        if: ${{ steps.tag.outputs.tag }}
+        id: upload
+        shell: bash
+        run: |
+          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
+          chmod 600 __TEMP_INPUT_KEY_FILE
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"

.github/workflows/node-tests.yml

@@ -0,0 +1,76 @@
+name: "node.js tests"
+
+# Cancel previously started workflow runs
+# when the branch is updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  tests:
+    name: Tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+      - name: System info
+        run: |
+          rustc -vV
+          rustup -vV
+          cargo -vV
+          npm --version
+          node --version
+
+      - name: Cache node modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ env.APPDATA }}/npm-cache
+            ~/.npm
+          key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry/
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2
+
+      - name: Install dependencies & build
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cd node
+          npm install --verbose
+
+      - name: Test
+        timeout-minutes: 10
+        if: runner.os != 'Windows'
+        run: |
+          cd node
+          npm run test
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"
+      - name: Run tests on Windows, except lint
+        timeout-minutes: 10
+        if: runner.os == 'Windows'
+        run: |
+          cd node
+          npm run test:mocha
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/repl.yml

@@ -11,22 +11,13 @@ jobs:
     name: Build REPL example
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@master
+      - uses: actions/checkout@v3
 
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: 1.50.0
-        override: true
+      - name: Build
+        run: cargo build -p deltachat-repl --features vendored
 
-    - name: build
-      uses: actions-rs/cargo@v1
-      with:
-        command: build
-        args: --example repl --features repl,vendored
-
-    - name: Upload binary
-      uses: actions/upload-artifact@v2
-      with:
-        name: repl.exe
-        path: 'target/debug/examples/repl.exe'
+      - name: Upload binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: repl.exe
+          path: "target/debug/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -0,0 +1,25 @@
+name: Build & Deploy Documentation on rs.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat --no-deps --document-private-items
+      - name: Upload to rs.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: ${{ secrets.USERNAME }}
+          KEY: ${{ secrets.KEY }}
+          HOST: "delta.chat"
+          SOURCE: "target/doc"
+          TARGET: "/var/www/html/rs/"

.github/workflows/upload-ffi-docs.yml

@@ -0,0 +1,25 @@
+name: Build & Deploy Documentation on cffi.delta.chat
+
+on:
+  push:
+    branches:
+      - master
+      - docs-gh-action
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build the documentation with cargo
+        run: |
+          cargo doc --package deltachat_ffi --no-deps
+      - name: Upload to cffi.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: ${{ secrets.USERNAME }}
+          KEY: ${{ secrets.KEY }}
+          HOST: "delta.chat"
+          SOURCE: "target/doc"
+          TARGET: "/var/www/html/cffi/"

.gitignore

@@ -12,8 +12,8 @@ include
 *.db
 *.db-blobs
 
+.tox
 python/.eggs
-python/.tox
 *.egg-info
 __pycache__
 python/src/deltachat/capi*.so
@@ -29,3 +29,14 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
+.vscode/launch.json
+python/accounts.txt
+python/all-testaccounts.txt
+tmp/
+
+# from deltachat-node
+node_modules/
+node/build/
+node/dist/
+node/prebuilds/
+node/.nyc_output/

.npmignore

@@ -0,0 +1,56 @@
+.circleci/
+.gitmodules
+node/.nyc_output/
+.travis.yml
+appveyor.yml
+node/build/
+node/README.md
+rustfmt.toml
+spec.md
+test-data/
+build2/
+node_modules
+.git
+.idea/
+.pytest_cache
+CMakeLists.txt
+README.md
+contrib/
+node/ci_scripts/
+coverage/
+node/.circleci
+node/appveyor.yml
+ci/
+ci_scripts/
+python/
+target
+proptest-regressions
+deltachat-ffi/Doxyfile
+scripts
+webxdc.md
+standards.md
+draft/
+node/examples/
+# deltachat-core-rust/assets # don't exclude assets, otherwise it won't build
+node/images/
+node/test/
+node/windows.md
+node/*.tar.gz
+node/old_docs.md
+.vscode/
+.github/
+node/.prettierrc.yml
+
+deltachat-jsonrpc/TODO.md
+deltachat-jsonrpc/README.MD
+deltachat-jsonrpc/.gitignore
+deltachat-jsonrpc/typescript/.gitignore
+deltachat-jsonrpc/typescript/.prettierignore
+deltachat-jsonrpc/typescript/accounts/
+deltachat-jsonrpc/typescript/index.html
+deltachat-jsonrpc/typescript/node-demo.js
+deltachat-jsonrpc/typescript/report_api_coverage.mjs
+deltachat-jsonrpc/typescript/test
+deltachat-jsonrpc/typescript/example.ts
+
+.DS_Store

.npmrc

@@ -0,0 +1 @@
+package-lock=false

CHANGELOG.md

@@ -1,5 +1,863 @@
 # Changelog
 
+## Unreleased
+
+### Changes
+- "full message view" not needed because of footers that go to contact status #4151
+- Pick up system's light/dark mode in generated message HTML #4150
+- Support non-persistent configuration with DELTACHAT_* env
+- Print deltachat-repl errors with causes. #4166
+- Increase MSRV to 1.64. #4167
+
+### Fixes
+- Fix segmentation fault if `dc_context_unref()` is called during
+  background process spawned by `dc_configure()` or `dc_imex()`
+  or `dc_jsonrpc_instance_t` is unreferenced
+  during handling the JSON-RPC request. #4153
+- Delete expired messages using multiple SQL requests. #4158
+- Do not emit "Failed to run incremental vacuum" warnings on success. #4160
+
+
+## 1.111.0
+
+### Changes
+- Make smeared timestamp generation non-async. #4075
+- Set minimum TLS version to 1.2. #4096
+- Run `cargo-deny` in CI. #4101
+- Check provider database with CI. #4099 
+- Switch to DEFERRED transactions #4100
+
+### Fixes
+- Do not block async task executor while decrypting the messages. #4079
+- Housekeeping: delete the blobs backup dir #4123
+
+### API-Changes
+- jsonrpc: add more advanced API to send a message. #4097
+- jsonrpc: add get webxdc blob API `getWebxdcBlob` #4070
+
+
+## 1.110.0
+
+### Changes
+- use transaction in `Contact::add_or_lookup()` #4059
+- Organize the connection pool as a stack rather than a queue to ensure that
+  connection page cache is reused more often.
+  This speeds up tests by 28%, real usage will have lower speedup. #4065
+- Use transaction in `update_blocked_mailinglist_contacts`. #4058
+- Remove `Sql.get_conn()` interface in favor of `.call()` and `.transaction()`. #4055
+- Updated provider database.
+- Disable DKIM-Checks again #4076
+- Switch from "X.Y.Z" and "py-X.Y.Z" to "vX.Y.Z" tags. #4089
+- mimeparser: handle headers from the signed part of unencrypted signed message #4013
+
+### Fixes
+- Start SQL transactions with IMMEDIATE behaviour rather than default DEFERRED one. #4063
+- Fix a problem with Gmail where (auto-)deleted messages would get archived instead of deleted.
+  Move them to the Trash folder for Gmail which auto-deletes trashed messages in 30 days #3972
+- Clear config cache after backup import. This bug sometimes resulted in the import to seemingly work at first. #4067
+- Update timestamps in `param` columns with transactions. #4083
+
+### API-Changes
+
+
+## 1.109.0
+
+### Changes
+- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
+
+### Fixes
+- deltachat-rpc-server: do not block stdin while processing the request. #4041
+  deltachat-rpc-server now reads the next request as soon as previous request handler is spawned.
+- Enable `auto_vacuum` on all SQL connections. #2955
+- Replace `r2d2` connection pool with an own implementation. #4050 #4053 #4043 #4061
+  This change improves reliability
+  by closing all database connections immediately when the context is closed.
+
+### API-Changes
+
+- Remove `MimeMessage::from_bytes()` public interface. #4033
+- BREAKING Types: jsonrpc: `get_messages` now returns a map with `MessageLoadResult` instead of failing completely if one of the requested messages could not be loaded. #4038
+- Add `dc_msg_set_subject()`. C-FFI #4057
+- Mark python bindings as supporting typing according to PEP 561 #4045
+
+
+## 1.108.0
+
+### Changes
+- Use read/write timeouts instead of per-command timeouts for SMTP #3985
+- Cache DNS results for SMTP connections #3985
+- Prefer TLS over STARTTLS during autoconfiguration #4021
+- Use SOCKS5 configuration for HTTP requests #4017
+- Show non-deltachat emails by default for new installations #4019
+- Re-enabled SMTP pipelining after disabling it in #4006
+
+### Fixes
+- Fix Securejoin for multiple devices on a joining side #3982
+- python: handle NULL value returned from `dc_get_msg()` #4020
+  Account.`get_message_by_id` may return `None` in this case.
+
+### API-Changes
+- Remove bitflags from `get_chat_msgs()` interface #4022
+  C interface is not changed.
+  Rust and JSON-RPC API have `flags` integer argument
+  replaced with two boolean flags `info_only` and `add_daymarker`.
+- jsonrpc: add API to check if the message is sent by a bot #3877
+
+
+## 1.107.1
+
+### Changes
+- Log server security (TLS/STARTTLS/plain) type #4005
+
+### Fixes
+- Disable SMTP pipelining #4006
+
+
+## 1.107.0
+
+### Changes
+- Pipeline SMTP commands #3924
+- Cache DNS results for IMAP connections #3970
+
+### Fixes
+- Securejoin: Fix adding and handling Autocrypt-Gossip headers #3914
+- fix verifier-by addr was empty string instead of None #3961
+- Emit DC_EVENT_MSGS_CHANGED for DC_CHAT_ID_ARCHIVED_LINK when the number of archived chats with
+  unread messages increases #3959
+- Fix Peerstate comparison #3962
+- Log SOCKS5 configuration for IMAP like already done for SMTP #3964
+- Fix SOCKS5 usage for IMAP #3965
+- Exit from recently seen loop on interrupt channel errors to avoid busy looping #3966
+
+### API-Changes
+- jsonrpc: add verified-by information to `Contact`-Object
+- Remove `attach_selfavatar` config #3951
+
+### Changes
+- add debug logging support for webxdcs #3296
+
+## 1.106.0
+
+### Changes
+- Only send IncomingMsgBunch if there are more than 0 new messages #3941
+
+### Fixes
+- fix: only send contact changed event for recently seen if it is relevant (not too old to matter) #3938
+- Immediately save `accounts.toml` if it was modified by a migration from absolute paths to relative paths #3943
+- Do not treat invalid email addresses as an exception #3942
+- Add timeouts to HTTP requests #3948
+
+
+## 1.105.0
+
+### Changes
+- Validate signatures in try_decrypt() even if the message isn't encrypted #3859
+- Don't parse the message again after detached signatures validation #3862
+- Move format=flowed support to a separate crate #3869
+- cargo: bump quick-xml from 0.23.0 to 0.26.0 #3722
+- Add fuzzing tests #3853
+- Add mappings for some file types to Viewtype / MIME type #3881
+- Buffer IMAP client writes #3888
+- move `DC_CHAT_ID_ARCHIVED_LINK` to the top of chat lists
+  and make `dc_get_fresh_msg_cnt()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3918
+- make `dc_marknoticed_chat()` work for `DC_CHAT_ID_ARCHIVED_LINK` #3919
+- Update provider database
+
+### API-Changes
+- jsonrpc: add python API for webxdc updates #3872
+- jsonrpc: add fresh message count to ChatListItemFetchResult::ArchiveLink
+- Add ffi functions to retrieve `verified by` information #3786
+- resultify `Message::get_filebytes()` #3925
+
+### Fixes
+- Do not add an error if the message is encrypted but not signed #3860
+- Do not strip leading spaces from message lines #3867
+- Fix uncaught exception in JSON-RPC tests #3884
+- Fix STARTTLS connection and add a test for it #3907
+- Trigger reconnection when failing to fetch existing messages #3911
+- Do not retry fetching existing messages after failure, prevents infinite reconnection loop #3913
+- Ensure format=flowed formatting is always reversible on the receiver side #3880
+
+
+## 1.104.0
+
+### Changes
+- Don't use deprecated `chrono` functions #3798
+- Document accounts manager #3837
+- If a classical-email-user sends an email to a group and adds new recipients,
+  add the new recipients as group members #3781
+- Remove `pytest-async` plugin #3846
+- Only send the message about ephemeral timer change if the chat is promoted #3847
+- Use relative paths in `accounts.toml` #3838
+
+### Fixes
+- Set read/write timeouts for IMAP over SOCKS5 #3833
+- Treat attached PGP keys as peer keys with mutual encryption preference #3832
+- fix migration of old databases #3842
+- Fix cargo clippy and doc errors after Rust update to 1.66 #3850
+- Don't send GroupNameChanged message if the group name doesn't change in terms of
+  `improve_single_line_input()` #3852
+- Prefer encryption for the peer if the message is encrypted or signed with the known key #3849
+
+
+## 1.103.0
+
+### Changes
+- Disable Autocrypt & Authres-checking for mailing lists,
+  because they don't work well with mailing lists #3765
+- Refactor: Remove the remaining AsRef<str> #3669
+- Add more logging to `fetch_many_msgs` and refactor it #3811
+- Small speedup #3780
+- Log the reason when the message cannot be sent to the chat #3810
+- Add IMAP server ID line to the context info only when it is known #3814
+- Remove autogenerated typescript files #3815
+- Move functions that require an IMAP session from `Imap` to `Session`
+  to reduce the number of code paths where IMAP session may not exist.
+  Drop connection on error instead of trying to disconnect,
+  potentially preventing IMAP task from getting stuck. #3812
+
+### API-Changes
+- Add Python API to send reactions #3762
+- jsonrpc: add message errors to MessageObject #3788
+- jsonrpc: Add async Python client #3734
+
+### Fixes
+- Make sure malformed messages will never block receiving further messages anymore #3771
+- strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650
+- Assume all Thunderbird users prefer encryption #3774
+- refactor peerstate handling to ensure no duplicate peerstates #3776
+- Fetch messages in order of their INTERNALDATE (fixes reactions for Gmail f.e.) #3789
+- python: do not pass NULL to ffi.gc if the context can't be created #3818
+- Add read/write timeouts to IMAP sockets #3820
+- Add connection timeout to IMAP sockets #3828
+- Disable read timeout during IMAP IDLE #3826
+- Bots automatically accept mailing lists #3831
+
+
+## 1.102.0
+
+### Changes
+
+- If an email has multiple From addresses, handle this as if there was
+  no From address, to prevent from forgery attacks. Also, improve
+  handling of emails with invalid From addresses in general #3667
+
+### API-Changes
+
+### Fixes
+- fix detection of "All mail", "Trash", "Junk" etc folders. #3760
+- fetch messages sequentially to fix reactions on partially downloaded messages #3688
+- Fix a bug where one malformed message blocked receiving any further messages #3769
+
+
+## 1.101.0
+
+### Changes
+- add `configured_inbox_folder` to account info #3748
+- `dc_delete_contact()` hides contacts if referenced #3751
+- add IMAP UIDs to message info #3755
+
+### Fixes
+- improve IMAP logging, in particular fix incorrect "IMAP IDLE protocol
+  timed out" message on network error during IDLE #3749
+- pop Recently Seen Loop event out of the queue when it is in the past
+  to avoid busy looping #3753
+- fix build failures by going back to standard `async_zip` #3747
+
+
+## 1.100.0
+
+### API-Changes
+- jsonrpc: add `miscSaveSticker` method
+
+### Changes
+- add JSON-RPC stdio server `deltachat-rpc-server` and use it for JSON-RPC tests #3695
+- update rPGP from 0.8 to 0.9 #3737
+- jsonrpc: typescript client: use npm released deltachat fork of the tiny emitter package #3741
+- jsonrpc: show sticker image in quote #3744
+
+
+
+## 1.99.0
+
+### API-Changes
+- breaking jsonrpc: changed function naming
+  - `autocryptInitiateKeyTransfer` -> `initiateAutocryptKeyTransfer`
+  - `autocryptContinueKeyTransfer` -> `continueAutocryptKeyTransfer`
+  - `chatlistGetFullChatById` -> `getFullChatById`
+  - `messageGetMessage` -> `getMessage`
+  - `messageGetMessages` -> `getMessages`
+  - `messageGetNotificationInfo` -> `getMessageNotificationInfo`
+  - `contactsGetContact` -> `getContact`
+  - `contactsCreateContact` -> `createContact`
+  - `contactsCreateChatByContactId` -> `createChatByContactId`
+  - `contactsBlock` -> `blockContact`
+  - `contactsUnblock` -> `unblockContact`
+  - `contactsGetBlocked` -> `getBlockedContacts`
+  - `contactsGetContactIds` -> `getContactIds`
+  - `contactsGetContacts` -> `getContacts`
+  - `contactsGetContactsByIds` -> `getContactsByIds`
+  - `chatGetMedia` -> `getChatMedia`
+  - `chatGetNeighboringMedia` -> `getNeighboringChatMedia`
+  - `webxdcSendStatusUpdate` -> `sendWebxdcStatusUpdate`
+  - `webxdcGetStatusUpdates` -> `getWebxdcStatusUpdates`
+  - `messageGetWebxdcInfo` -> `getWebxdcInfo`
+- jsonrpc: changed method signature
+  - `miscSendTextMessage(accountId, text, chatId)` -> `miscSendTextMessage(accountId, chatId, text)`
+- jsonrpc: add `SystemMessageType` to `Message`
+- cffi: add missing `DC_INFO_` constants
+- Add DC_EVENT_INCOMING_MSG_BUNCH event #3643
+- Python bindings: Make get_matching() only match the
+  whole event name, e.g. events.get_matching("DC_EVENT_INCOMING_MSG")
+  won't match DC_EVENT_INCOMING_MSG_BUNCH anymore #3643
+
+
+- Rust: Introduce a ContextBuilder #3698
+
+### Changes
+- allow sender timestamp to be in the future, but not too much
+- Disable the new "Authentication-Results/DKIM checking" security feature
+  until we have tested it a bit #3728
+- refactorings #3706
+
+### Fixes
+- `dc_search_msgs()` returns unaccepted requests #3694
+- emit "contacts changed" event when the contact is no longer "seen recently" #3703
+- do not allow peerstate reset if DKIM check failed #3731
+
+
+## 1.98.0
+
+### API-Changes
+- jsonrpc: typescript client: export constants under `C` enum, similar to how its exported from `deltachat-node` #3681
+- added reactions support #3644
+- jsonrpc: reactions: added reactions to `Message` type and the `sendReaction()` method #3686
+
+### Changes
+- simplify `UPSERT` queries #3676
+
+### Fixes
+
+
+## 1.97.0
+
+### API-Changes
+- jsonrpc: add function: #3641, #3645, #3653
+  - `getChatContacts()`
+  - `createGroupChat()`
+  - `createBroadcastList()`
+  - `setChatName()`
+  - `setChatProfileImage()`
+  - `downloadFullMessage()`
+  - `lookupContactIdByAddr()`
+  - `sendVideochatInvitation()`
+  - `searchMessages()`
+  - `messageIdsToSearchResults()`
+  - `setChatVisibility()`
+  - `getChatEphemeralTimer()`
+  - `setChatEphemeralTimer()`
+  - `getLocations()`
+  - `getAccountFileSize()`
+  - `estimateAutoDeletionCount()`
+  - `setStockStrings()`
+  - `exportSelfKeys()`
+  - `importSelfKeys()`
+  - `sendSticker()`
+  - `changeContactName()`
+  - `deleteContact()`
+  - `joinSecurejoin()`
+  - `stopIoForAllAccounts()`
+  - `startIoForAllAccounts()`
+  - `startIo()`
+  - `stopIo()`
+  - `exportBackup()`
+  - `importBackup()`
+  - `getMessageHtml()` #3671
+  - `miscGetStickerFolder` and `miscGetStickers` #3672
+- breaking: jsonrpc: remove function `messageListGetMessageIds()`, it is replaced by `getMessageIds()` and `getMessageListItems()` the latter returns a new `MessageListItem` type, which is the now preferred way of using the message list.
+- jsonrpc: add type: #3641, #3645
+  - `MessageSearchResult`
+  - `Location`
+- jsonrpc: add `viewType` to quoted message(`MessageQuote` type) in `Message` object type #3651
+
+
+### Changes
+- Look at Authentication-Results. Don't accept Autocrypt key changes
+  if they come with negative authentiation results while this contact
+  sent emails with positive authentication results in the past. #3583
+- jsonrpc in cffi also sends events now #3662
+- jsonrpc: new format for events and better typescript autocompletion
+- Join all "[migration] vXX" log messages into one
+
+### Fixes
+- share stock string translations across accounts created by the same account manager #3640
+- suppress welcome device messages after account import #3642
+- fix unix timestamp used for daymarker #3660
+
+## 1.96.0
+
+### Changes
+- jsonrpc js client:
+  - Change package name from `deltachat-jsonrpc-client` to `@deltachat/jsonrpc-client`
+  - remove relative file dependency to it from `deltachat-node` (because it did not work anyway and broke the nix build of desktop)
+  - ci: add github ci action to upload it to our download server automatically on release
+
+## 1.95.0
+
+### API-Changes
+- jsonrpc: add `mailingListAddress` property to `FullChat` #3607
+- jsonrpc: add `MessageNotificationInfo` & `messageGetNotificationInfo()` #3614
+- jsonrpc: add `chat_get_neighboring_media` function #3610
+
+### Changes
+- added `dclogin:` scheme to allow configuration from a qr code
+  (data inside qrcode, contrary to `dcaccount:` which points to an API to create an account) #3541
+- truncate incoming messages by lines instead of just length #3480
+- emit separate `DC_EVENT_MSGS_CHANGED` for each expired message,
+  and `DC_EVENT_WEBXDC_INSTANCE_DELETED` when a message contains a webxdc #3605
+- enable `bcc_self` by default #3612
+
+
+## 1.94.0
+
+### API-Changes
+- breaking change: replace `dc_accounts_event_emitter_t` with `dc_event_emitter_t` #3422
+
+  Type `dc_accounts_event_emitter_t` is removed.
+  `dc_accounts_get_event_emitter()` returns `dc_event_emitter_t` now, so
+  `dc_get_next_event()` should be used instead of `dc_accounts_get_next_event`
+  and `dc_event_emitter_unref()` should be used instead of
+  `dc_accounts_event_emitter_unref`.
+- add `dc_contact_was_seen_recently()` #3560
+- Fix `get_connectivity_html` and `get_encrinfo` futures not being Send. See rust-lang/rust#101650 for more information
+- jsonrpc: add functions: #3586, #3587, #3590
+  - `deleteChat()`
+  - `getChatEncryptionInfo()`
+  - `getChatSecurejoinQrCodeSvg()`
+  - `leaveGroup()`
+  - `removeContactFromChat()`
+  - `addContactToChat()`
+  - `deleteMessages()`
+  - `getMessageInfo()`
+  - `getBasicChatInfo()`
+  - `marknoticedChat()`
+  - `getFirstUnreadMessageOfChat()`
+  - `markseenMsgs()`
+  - `forwardMessages()`
+  - `removeDraft()`
+  - `getDraft()`
+  - `miscSendMsg()`
+  - `miscSetDraft()`
+  - `maybeNetwork()`
+  - `getConnectivity()`
+  - `getContactEncryptionInfo()`
+  - `getConnectivityHtml()`
+- jsonrpc: add `is_broadcast` property to `ChatListItemFetchResult` #3584
+- jsonrpc: add `was_seen_recently` property to `ChatListItemFetchResult`, `FullChat` and `Contact` #3584
+- jsonrpc: add `webxdc_info` property to `Message` #3588
+- python: move `get_dc_event_name()` from `deltachat` to `deltachat.events` #3564
+- jsonrpc: add `webxdc_info`, `parent_id` and `download_state` property to `Message` #3588, #3590
+- jsonrpc: add `BasicChat` object as a leaner alternative to `FullChat` #3590
+- jsonrpc: add `last_seen` property to `Contact` #3590
+- breaking! jsonrpc: replace `Message.quoted_text` and `Message.quoted_message_id` with `Message.quote` #3590
+- add separate stock strings for actions done by contacts to make them easier to translate #3518
+- `dc_initiate_key_transfer()` is non-blocking now. #3553
+  UIs don't need to display a button to cancel sending Autocrypt Setup Message with
+  `dc_stop_ongoing_process()` anymore.
+
+### Changes
+- order contact lists by "last seen";
+  this affects `dc_get_chat_contacts()`, `dc_get_contacts()` and `dc_get_blocked_contacts()` #3562
+- add `internet_access` flag to `dc_msg_get_webxdc_info()` #3516
+- `DC_EVENT_WEBXDC_INSTANCE_DELETED` is emitted when a message containing a webxdc gets deleted #3592
+
+### Fixes
+- do not emit notifications for blocked chats #3557
+- Show attached .eml files correctly #3561
+- Auto accept contact requests if `Config::Bot` is set for a client #3567 
+- Don't prepend the subject to chat messages in mailinglists
+- fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585
+- Reject webxdc-updates from contacts who are not group members #3568
+
+
+## 1.93.0
+
+### API-Changes
+- added a JSON RPC API, accessible through a WebSocket server, the CFFI bindings and the Node.js bindings #3463 #3554 #3542
+- JSON RPC methods in CFFI #3463:
+ - `dc_jsonrpc_instance_t* dc_jsonrpc_init(dc_accounts_t* account_manager);`
+ - `void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);`
+ - `void dc_jsonrpc_request(dc_jsonrpc_instance_t* jsonrpc_instance, char* request);`
+ - `char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);`
+- node: JSON RPC methods #3463:
+ - `AccountManager.prototype.startJsonRpcHandler(callback: ((response: string) => void)): void`
+ - `AccountManager.prototype.jsonRpcRequest(message: string): void`
+
+### Changes
+- use [pathlib](https://docs.python.org/3/library/pathlib.html) in provider update script #3543
+- `dc_get_chat_media()` can return media globally #3528
+- node: add `getMailinglistAddr()` #3524
+- avoid duplicate encoded-words package and test `cargo vendor` in ci #3549
+- python: don't raise an error if addr changes #3530
+- improve coverage script #3530
+
+### Fixes
+- improved error handling for account setup from qrcode #3474
+- python: enable certificate checks in cloned accounts #3443
+
+
+## 1.92.0
+
+### API-Changes
+- add `dc_chat_get_mailinglist_addr()` #3520
+
+
+## 1.91.0
+
+### Added
+- python bindings: extra method to get an account running
+
+### Changes
+- refactorings #3437
+
+### Fixes
+- mark "group image changed" as system message on receiver side #3517
+
+
+## 1.90.0
+
+### Changes
+- handle drafts from mailto links in scanned QR #3492
+- do not overflow ratelimiter leaky bucket #3496
+- (AEAP) Add device message after you changed your address #3505
+- (AEAP) Revert #3491, instead only replace contacts in verified groups #3510
+- improve python bindings and tests #3502 #3503
+
+### Fixes
+- don't squash text parts of NDN into attachments #3497
+- do not treat non-failed DSNs as NDNs #3506
+
+
+## 1.89.0
+
+### Changes
+
+- (AEAP) When one of your contacts changed their address, they are
+  only replaced in the chat where you got a message from them
+  for now #3491
+
+### Fixes
+- replace musl libc name resolution errors with a better message #3485
+- handle updates for not yet downloaded webxdc instances #3487
+
+
+## 1.88.0
+
+### Changes
+- Implemented "Automatic e-mail address Porting" (AEAP). You can
+  configure a new address in DC now, and when receivers get messages
+  they will automatically recognize your moving to a new address. #3385
+- switch from `async-std` to `tokio` as the async runtime #3449
+- upgrade to `pgp@0.8.0` #3467
+- add IMAP ID extension support #3468
+- configure DeltaChat folder by selecting it, so it is configured even if not LISTed #3371
+- build PyPy wheels #6683
+- improve default error if NDN does not provide an error #3456
+- increase ratelimit from 3 to 6 messages per 60 seconds #3481
+
+### Fixes
+- mailing list: remove square-brackets only for first name #3452
+- do not use footers from mailinglists as the contact status #3460
+- don't ignore KML parsing errors #3473
+
+
+## 1.87.0
+
+### Changes
+- limit the rate of MDN sending #3402
+- ignore ratelimits for bots #3439
+- remove `msgs_mdns` references to deleted messages during housekeeping #3387
+- format message lines starting with `>` as quotes #3434
+- node: remove `split2` dependency #3418
+- node: add git installation info to readme #3418
+- limit the rate of webxdc update sending #3417
+
+### Fixes
+- set a default error if NDN does not provide an error #3410
+- python: avoid exceptions when messages/contacts/chats are compared with `None`
+- node: wait for the event loop to stop before destroying contexts #3431 #3451
+- emit configuration errors via event on failure #3433
+- report configure and imex success/failure after freeing ongoing process #3442
+
+### API-Changes
+- python: added `Message.get_status_updates()`  #3416
+- python: added `Message.send_status_update()`  #3416
+- python: added `Message.is_webxdc()`  #3416
+- python: added `Message.is_videochat_invitation()`  #3416
+- python: added support for "videochat" and "webxdc" view types to `Message.new_empty()`  #3416
+
+
+## 1.86.0
+
+### API-Changes
+- python: added optional `closed` parameter to `Account` constructor #3394
+- python: added optional `passphrase` parameter to `Account.export_all()` and `Account.import_all()` #3394
+- python: added `Account.open()` #3394
+- python: added `Chat.is_single()` #3394
+- python: added `Chat.is_mailinglist()` #3394
+- python: added `Chat.is_broadcast()` #3394
+- python: added `Chat.is_multiuser()` #3394
+- python: added `Chat.is_self_talk()` #3394
+- python: added `Chat.is_device_talk()` #3394
+- python: added `Chat.is_pinned()` #3394
+- python: added `Chat.pin()` #3394
+- python: added `Chat.unpin()` #3394
+- python: added `Chat.archive()` #3394
+- python: added `Chat.unarchive()` #3394
+- python: added `Message.get_summarytext()` #3394
+- python: added optional `closed` parameter to `ACFactory.get_unconfigured_account()` (pytest plugin) #3394
+- python: added optional `passphrase` parameter to `ACFactory.get_pseudo_configured_account()` (pytest plugin) #3394
+
+### Changes
+- clean up series of webxdc info messages;
+  `DC_EVENT_MSGS_CHANGED` is emitted on changes of existing info messages #3395
+- update provider database #3399
+- refactorings #3375 #3403 #3398 #3404
+
+### Fixes
+- do not reset our database if imported backup cannot be decrypted #3397
+- node: remove `npx` from build script, this broke flathub build #3396
+
+
+## 1.85.0
+
+### Changes
+- refactorings #3373 #3345 #3380 #3382
+- node: move split2 to devDependencies
+- python: build Python 3.10 wheels #3392
+- update Rust dependencies
+
+### Fixes
+- delete outgoing MDNs found in the Sent folder on Gmail #3372
+- fix searching one-to-one chats #3377
+- do not add legacy info-messages on resending webxdc #3389
+
+
+## 1.84.0
+
+### Changes
+- refactorings #3354 #3347 #3353 #3346
+
+### Fixes
+- do not unnecessarily SELECT folders if there are no operations planned on
+  them #3333
+- trim chat encryption info #3350
+- fix failure to decrypt first message to self after key synchronization
+  via Autocrypt Setup Message #3352
+- Keep pgp key when you change your own email address #3351
+- Do not ignore Sent and Spam folders on Gmail #3369
+- handle decryption errors explicitly and don't get confused by encrypted mail attachments #3374
+
+
+## 1.83.0
+
+### Fixes
+- fix node prebuild & package ci #3337
+
+
+## 1.82.0
+
+### API-Changes
+- re-add removed `DC_MSG_ID_MARKER1` as in use on iOS #3330
+
+### Changes
+- refactorings #3328
+
+### Fixes
+- fix node package ci #3331
+- fix race condition in ongoing process (import/export, configuration) allocation #3322
+
+
+## 1.81.0
+
+### API-Changes
+- deprecate unused `marker1before` argument of `dc_get_chat_msgs`
+  and remove `DC_MSG_ID_MARKER1` constant #3274
+
+### Changes
+- now the node-bindings are also part of this repository 🎉 #3283
+- support `source_code_url` from Webxdc manifests #3314
+- support Webxdc document names and add `document` to `dc_msg_get_webxdc_info()` #3317 #3324
+- improve chat encryption info, make it easier to find contacts without keys #3318
+- improve error reporting when creating a folder fails #3325
+- node: remove unmaintained coverage scripts
+- send normal messages with higher priority than MDNs #3243
+- make Scheduler stateless #3302
+- abort instead of unwinding on panic #3259
+- improve python bindings #3297 #3298
+- improve documentation #3307 #3306 #3309 #3319 #3321
+- refactorings #3304 #3303 #3323
+
+### Fixes
+- node: throw error when getting context with an invalid account id
+- node: throw error when instantiating a wrapper class on `null` (Context, Message, Chat, ChatList and so on)
+- use same contact-color if email address differ only in upper-/lowercase #3327
+- repair encrypted mails "mixed up" by Google Workspace "Append footer" function #3315
+
+
+## 1.80.0
+
+### Changes
+- update provider database #3284
+- improve python bindings, tests and ci #3287 #3286 #3287 #3289 #3290 #3292
+
+### Fixes
+- fix escaping in generated QR-code-SVG #3295
+
+
+## 1.79.0
+
+### Changes
+- Send locations in the background regardless of SMTP loop activity #3247
+- refactorings #3268
+- improve tests and ci #3266 #3271
+
+### Fixes
+- simplify `dc_stop_io()` and remove potential panics and race conditions #3273
+- fix correct message escaping consisting of a dot in SMTP protocol #3265
+
+
+## 1.78.0
+
+### API-Changes
+- replaced stock string `DC_STR_ONE_MOMENT` by `DC_STR_NOT_CONNECTED` #3222
+- add `dc_resend_msgs()` #3238
+- `dc_provider_new_from_email()` does no longer do an DNS lookup for checking custom domains,
+  this is done by `dc_provider_new_from_email_with_dns()` now #3256
+
+### Changes
+- introduce multiple self addresses with the "configured" address always being the primary one #2896
+- Further improve finding the correct server after logging in #3208
+- `get_connectivity_html()` returns HTML as non-scalable #3213
+- add update-serial to `DC_EVENT_WEBXDC_STATUS_UPDATE` #3215
+- Speed up message receiving via IMAP a bit #3225
+- mark messages as seen on IMAP in batches #3223
+- remove Received: based draft detection heuristic #3230
+- Use pkgconfig for building Python package #2590
+- don't start io on unconfigured context #2664
+- do not assign group IDs to ad-hoc groups #2798
+- dynamic libraries use dylib extension on Darwin #3226
+- refactorings #3217 #3219 #3224 #3235 #3239 #3244 #3254
+- improve documentation #3214 #3220 #3237
+- improve tests and ci #3212 #3233 #3241 #3242 #3252 #3250 #3255 #3260
+
+### Fixes
+- Take `delete_device_after` into account when calculating ephemeral loop timeout #3211 #3221
+- Fix a bug where a blocked contact could send a contact request #3218
+- Make sure, videochat-room-names are always URL-safe #3231
+- Try removing account folder multiple times in case of failure #3229
+- Ignore messages from all spam folders if there are many #3246
+- Hide location-only messages instead of displaying empty bubbles #3248
+
+
+## 1.77.0
+
+### API changes
+- change semantics of `dc_get_webxdc_status_updates()` second parameter
+  and remove update-id from `DC_EVENT_WEBXDC_STATUS_UPDATE` #3081
+
+### Changes
+- add more SMTP logging #3093
+- place common headers like `From:` before the large `Autocrypt:` header #3079
+- keep track of securejoin joiner status in database to survive restarts #2920
+- remove never used `SentboxMove` option #3111
+- improve speed by caching config values #3131 #3145
+- optimize `markseen_msgs` #3141
+- automatically accept chats with outgoing messages #3143
+- `dc_receive_imf` refactorings #3154 #3156 #3159
+- add index to speedup deletion of expired ephemeral messages #3155
+- muted chats stay archived on new messages #3184
+- support `min_api` from Webxdc manifests #3206
+- do not read whole webxdc file into memory #3109
+- improve tests, refactorings #3073 #3096 #3102 #3108 #3139 #3128 #3133 #3142 #3153 #3151 #3174 #3170 #3148 #3179 #3185
+- improve documentation #2983 #3112 #3103 #3118 #3120
+
+### Fixes
+- speed up loading of chat messages by a factor of 20 #3171 #3194 #3173
+- fix an issue where the app crashes when trying to export a backup #3195
+- hopefully fix a bug where outgoing messages appear twice with Amazon SES #3077
+- do not delete messages without Message-IDs as duplicates #3095
+- assign replies from a different email address to the correct chat #3119
+- assign outgoing private replies to the correct chat #3177
+- start ephemeral timer when seen status is synchronized via IMAP #3122
+- do not create empty contact requests with "setup changed" messages;
+  instead, send a "setup changed" message into all chats we share with the peer #3187
+- do not delete duplicate messages on IMAP immediately to accidentally deleting
+  the last copy #3138
+- clear more columns when message expires due to `delete_device_after` setting #3181
+- do not try to use stale SMTP connections #3180
+- slightly improve finding the correct server after logging in #3207
+- retry message sending automatically if loop is not interrupted #3183
+- fix a bug where sometimes the file extension of a long filename containing a dot was cropped #3098
+
+
+## 1.76.0
+
+### Changes
+- move messages in batches #3058
+- delete messages in batches #3060
+- python: remove arbitrary timeouts from tests #3059
+- refactorings #3026
+
+### Fixes
+- avoid archived, fresh chats #3053
+- Also resync UIDs in folders that are not configured #2289
+- treat "NO" IMAP response to MOVE and COPY commands as an error #3058
+- Fix a bug where messages in the Spam folder created contact requests #3015
+- Fix a bug where drafts disappeared after some days #3067
+- Parse MS Exchange read receipts and mark the original message as read #3075
+- do not retry message sending infinitely in case of permanent SMTP failure #3070
+- set message state to failed when retry limit is exceeded #3072
+
+
+## 1.75.0
+
+### Changes
+- optimize `delete_expired_imap_messages()` #3047
+
+
+## 1.74.0
+
+### Fixes
+- avoid reconnection loop when message without Message-ID is marked as seen #3044
+
+
+## 1.73.0
+
+### API changes
+- added `only_fetch_mvbox` config #3028
+
+### Changes
+- don't watch Sent folder by default #3025
+- use webxdc app name in chatlist/quotes/replies etc. #3027
+- make it possible to cancel message sending by removing the message #3034,
+  this was previously removed in 1.71.0 #2939
+- synchronize Seen flags only on watched folders to speed up
+  folder scanning #3041
+- remove direct dependency on `byteorder` crate #3031
+- refactorings #3023 #3013
+- update provider database #3043
+- improve documentation #3017 #3018 #3021
+
+### Fixes
+- fix splitting off text from webxdc messages #3032
+- call slow `delete_expired_imap_messages()` less often #3037
+- make synchronization of Seen status more robust in case unsolicited FETCH
+  result without UID is returned #3022
+- fetch Inbox before scanning folders to ensure iOS does
+  not kill the app before it gets to fetch the Inbox in background #3040
+
+
 ## 1.72.0
 
 ### Fixes
@@ -1205,7 +2063,7 @@
 - #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
 
 - #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
-  improvments 
+  improvements
 
 ## 1.0.0-beta.16
 
@@ -1284,7 +2142,7 @@
 - trigger reconnect more often on imap error states.  Should fix an 
   issue observed when trying to empty a folder.  @hpk42
 
-- un-split qr tests: we fixed qr-securejoin protocol flakyness 
+- un-split qr tests: we fixed qr-securejoin protocol flakiness 
   last weeks. @hpk42
 
 ## 1.0.0-beta.10
@@ -1322,7 +2180,7 @@
 
 - fix moving self-sent messages, thanks @r10s, @flub, @hpk42
 
-- fix flakyness/sometimes-failing verified/join-protocols, 
+- fix flakiness/sometimes-failing verified/join-protocols, 
   thanks @flub, @r10s, @hpk42
 
 - fix reply-to-encrypted message to keep encryption 
@@ -1341,7 +2199,7 @@
 
 - fixes imap-protocol parsing bugs that lead to infinitely
   repeated crashing while trying to receive messages with
-  a subjec that contained non-utf8. thanks @link2xt
+  a subject that contained non-utf8. thanks @link2xt
 
 - fixed logic to find encryption subkey -- previously 
   delta chat would use the primary key for encryption

CMakeLists.txt

@@ -4,16 +4,24 @@ include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
+if(APPLE)
+       set(DYNAMIC_EXT "dylib")
+elseif(UNIX)
+       set(DYNAMIC_EXT "so")
+else()
+       set(DYNAMIC_EXT "dll")
+endif()
+
 add_custom_command(
 	OUTPUT
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
+        ${CARGO} build --release --no-default-features --features jsonrpc
 
 	# Build in `deltachat-ffi` directory instead of using
 	# `--package deltachat_ffi` to avoid feature resolver version
@@ -32,11 +40,11 @@ add_custom_target(
 	ALL
 	DEPENDS
 	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.so"
+	"target/release/libdeltachat.${DYNAMIC_EXT}"
 	"target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,111 +1,116 @@
 [package]
 name = "deltachat"
-version = "1.72.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
-edition = "2018"
+version = "1.111.0"
+edition = "2021"
 license = "MPL-2.0"
-resolver = "2"
+rust-version = "1.64"
 
 [profile.dev]
 debug = 0
+panic = 'abort'
+opt-level = 1
+
+[profile.test]
+opt-level = 0
+
+# Always optimize dependencies.
+# This does not apply to crates in the workspace.
+# <https://doc.rust-lang.org/cargo/reference/profiles.html#overrides>
+[profile.dev.package."*"]
+opt-level = "z"
 
 [profile.release]
 lto = true
-
-[patch.crates-io]
-rusqlite = { git = "https://github.com/rusqlite/rusqlite", branch="master" }
+panic = 'abort'
+opt-level = "z"
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+format-flowed = { path = "./format-flowed" }
+ratelimit = { path = "./deltachat-ratelimit" }
 
-ansi_term = { version = "0.12.1", optional = true }
 anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
+async-channel = "1.8.0"
+async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
+async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
 backtrace = "0.3"
-base64 = "0.13"
+base64 = "0.21"
 bitflags = "1.3"
-byteorder = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
+chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
+fast-socks5 = "0.8"
 futures = "0.3"
+futures-lite = "1.12.0"
 hex = "0.4.0"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+humansize = "2"
+image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-log = {version = "0.4.8", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
+mailparse = "0.14"
+num_cpus = "1.15"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.9.0"
-percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
+once_cell = "1.17.0"
+percent-encoding = "2.2"
+parking_lot = "0.12"
+pgp = { version = "0.9", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.19"
-rand = "0.7"
-regex = "1.5"
-rusqlite = { version = "0.26", features = ["sqlcipher"] }
+qrcodegen = "1.7.0"
+quick-xml = "0.27"
+rand = "0.8"
+regex = "1.7"
+reqwest = { version = "0.11.14", features = ["json"] }
+rusqlite = { version = "0.28", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
+sanitize-filename = "0.4"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-stop-token = "0.7"
-strum = "0.23"
-strum_macros = "0.23"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
+strum = "0.24"
+strum_macros = "0.24"
+tagger = "4.3.4"
+textwrap = "0.16.0"
 thiserror = "1"
-toml = "0.5"
+tokio-io-timeout = "1.2.0"
+tokio-stream = { version = "0.1.11", features = ["fs"] }
+tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+toml = "0.7"
+trust-dns-resolver = "0.22"
 url = "2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
-qrcodegen = "1.7.0"
-tagger = "4.2.1"
-textwrap = "0.14.2"
-zip = { version = "0.5.13", default-features = false, features = ["deflate"] }
+uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1", features = ["unstable", "attributes"] }
-criterion = "0.3"
+criterion = { version = "0.4.0", features = ["async_tokio"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
+tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 
 [workspace]
 members = [
   "deltachat-ffi",
   "deltachat_derive",
+  "deltachat-jsonrpc",
+  "deltachat-rpc-server",
+  "deltachat-ratelimit",
+  "deltachat-repl",
+  "format-flowed",
 ]
 
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
-required-features = ["repl"]
-
-[[example]]
-name = "repl"
-path = "examples/repl/main.rs"
-required-features = ["repl"]
 
 
 [[bench]]
@@ -120,9 +125,27 @@ harness = false
 name = "search_msgs"
 harness = false
 
+[[bench]]
+name = "receive_emails"
+harness = false
+
+[[bench]]
+name = "get_chat_msgs"
+harness = false
+
+[[bench]]
+name = "get_chatlist"
+harness = false
+
+[[bench]]
+name = "send_events"
+harness = false
+
 [features]
 default = ["vendored"]
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled-sqlcipher-vendored-openssl"]
-nightly = ["pgp/nightly"]
+vendored = [
+  "async-native-tls/vendored",
+  "rusqlite/bundled-sqlcipher-vendored-openssl",
+  "reqwest/native-tls-vendored"
+]

README.md

@@ -12,15 +12,26 @@ To download and install the official compiler for the Rust programming language,
 $ curl https://sh.rustup.rs -sSf | sh
 ```
 
+> On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
+
 ## Using the CLI client
 
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=repl=info cargo run --example repl --features repl -- ~/deltachat-db
+$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
+Optionally, install `deltachat-repl` binary with
+```
+$ cargo install --path deltachat-repl/
+```
+and run as
+```
+$ deltachat-repl ~/deltachat-db
+```
+
 Configure your account (if not already configured):
 
 ```
@@ -100,12 +111,9 @@ $ cargo build -p deltachat_ffi --release
 
 ## Debugging environment variables 
 
-- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
-  printed
-
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests
@@ -116,20 +124,56 @@ use the `--ignored` argument to the test binary (not to cargo itself):
 $ cargo test -- --ignored
 ```
 
+### Fuzzing
+
+Install [`cargo-bolero`](https://github.com/camshaft/bolero) with
+```sh
+$ cargo install cargo-bolero
+```
+
+Run fuzzing tests with
+```sh
+$ cd fuzz
+$ cargo bolero test fuzz_mailparse --release=false -s NONE
+```
+
+Corpus is created at `fuzz/fuzz_targets/corpus`,
+you can add initial inputs there.
+For `fuzz_mailparse` target corpus can be populated with
+`../test-data/message/*.eml`.
+
+To run with AFL instead of libFuzzer:
+```sh
+$ cargo bolero test fuzz_format_flowed --release=false -e afl -s NONE
+```
+
 ## Features
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
 
+## Update Provider Data
+
+To add the updates from the
+[provider-db](https://github.com/deltachat/provider-db) to the core, run:
+
+```
+./src/provider/update.py ../provider-db/_providers/ > src/provider/data.rs
+```
+
 ## Language bindings and frontend projects
 
 Language bindings are available for:
 
 - **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
-- **Node.js** \[[📂 source](https://github.com/deltachat/deltachat-node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
+- **Node.js** 
+  - over cffi (legacy): \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
+  - over jsonrpc built with napi.rs: \[[📂 source](https://github.com/deltachat/napi-jsonrpc) | [📦 npm](https://www.npmjs.com/package/@deltachat/napi-jsonrpc)\]
 - **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
-- **Go** \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
-- **Free Pascal** \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
+- **Go**
+  - over jsonrpc: \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]
+  - over cffi[^1]: \[[📂 source](https://github.com/deltachat/go-deltachat/)\]
+- **Free Pascal**[^1] \[[📂 source](https://github.com/deltachat/deltachat-fp/)\]
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -141,3 +185,5 @@ or its language bindings:
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
 - [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**
+
+[^1]: Out of date / unmaintained, if you like those languages feel free to start maintaining them. If you have questions we'll help you, please ask in the issues.

assets/icon-archive.svg

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="60"
+   height="60"
+   viewBox="0 0 60 60"
+   fill="none"
+   stroke="currentColor"
+   stroke-width="2"
+   stroke-linecap="round"
+   stroke-linejoin="round"
+   class="feather feather-archive"
+   version="1.1"
+   id="svg8"
+   sodipodi:docname="icon-archive.svg"
+   inkscape:version="1.2.2 (b0a84865, 2022-12-01)"
+   inkscape:export-filename="icon-archive.png"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-ydpi="409.60001"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs12" />
+  <sodipodi:namedview
+     id="namedview10"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     showgrid="false"
+     inkscape:zoom="6.4597151"
+     inkscape:cx="24.459283"
+     inkscape:cy="32.509174"
+     inkscape:window-width="1457"
+     inkscape:window-height="860"
+     inkscape:window-x="55"
+     inkscape:window-y="38"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg8" />
+  <g
+     id="g846"
+     transform="translate(0.558605,0.464417)">
+    <path
+       style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="M 38.749006,25.398867 V 38.843194 H 20.133784 V 25.398867"
+       id="path847" />
+    <path
+       style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="m 18.065427,20.227972 h 22.751936 v 5.170894 H 18.065427 Z"
+       id="path845" />
+    <path
+       style="fill:#ff0000;fill-opacity:1;stroke:#808080;stroke-width:1.78186;stroke-dasharray:none;stroke-opacity:1"
+       d="m 27.373036,29.535581 h 4.136718"
+       id="line6" />
+  </g>
+</svg>

benches/contacts.rs

@@ -1,17 +1,20 @@
-use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::contact::Contact;
 use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::Events;
 use tempfile::tempdir;
 
 async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(&dbfile, id, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
 
     let book = (0..n)
-        .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))
+        .map(|i| format!("Name {i}\naddr{i}@example.org\n"))
         .collect::<Vec<String>>()
         .join("");
 
@@ -24,12 +27,16 @@ async fn address_book_benchmark(n: u32, read_count: u32) {
 }
 
 fn criterion_benchmark(c: &mut Criterion) {
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
     c.bench_function("create 500 contacts", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(500), black_box(0)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(500), black_box(0)).await })
     });
 
     c.bench_function("create 100 contacts and read it 1000 times", |b| {
-        b.iter(|| block_on(async { address_book_benchmark(black_box(100), black_box(1000)).await }))
+        b.to_async(&rt)
+            .iter(|| async { address_book_benchmark(black_box(100), black_box(1000)).await })
     });
 }
 

benches/create_account.rs

@@ -1,12 +1,12 @@
-use async_std::path::PathBuf;
-use async_std::task::block_on;
+use std::path::PathBuf;
+
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::accounts::Accounts;
 use tempfile::tempdir;
 
 async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
-    let p: PathBuf = dir.path().join("accounts").into();
+    let p: PathBuf = dir.path().join("accounts");
 
     let mut accounts = Accounts::new(p.clone()).await.unwrap();
 
@@ -18,7 +18,8 @@ async fn create_accounts(n: u32) {
 
 fn criterion_benchmark(c: &mut Criterion) {
     c.bench_function("create 1 account", |b| {
-        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        b.to_async(&rt).iter(|| create_accounts(black_box(1)))
     });
 }
 

benches/get_chat_msgs.rs

@@ -0,0 +1,46 @@
+use std::path::Path;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::chat::{self, ChatId};
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::Events;
+
+async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
+    let id = 100;
+    let context = Context::new(dbfile, id, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
+
+    for c in chats.iter().take(10) {
+        black_box(chat::get_chat_msgs(&context, *c).await.ok());
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
+        let chats: Vec<_> = rt.block_on(async {
+            let context = Context::new(Path::new(&path), 100, Events::new(), StockStrings::new())
+                .await
+                .unwrap();
+            let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
+            let len = chatlist.len();
+            (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()
+        });
+
+        c.bench_function("chat::get_chat_msgs (load messages from 10 chats)", |b| {
+            b.to_async(&rt)
+                .iter(|| get_chat_msgs_benchmark(black_box(path.as_ref()), black_box(&chats)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/get_chatlist.rs

@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::chatlist::Chatlist;
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::Events;
+
+async fn get_chat_list_benchmark(context: &Context) {
+    Chatlist::try_load(context, 0, None, None).await.unwrap();
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
+    // messages, such as your primary account.
+    if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(async {
+            Context::new(Path::new(&path), 100, Events::new(), StockStrings::new())
+                .await
+                .unwrap()
+        });
+        c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
+            b.to_async(&rt)
+                .iter(|| get_chat_list_benchmark(black_box(&context)))
+        });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
+    }
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/receive_emails.rs

@@ -0,0 +1,157 @@
+use std::path::PathBuf;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::{
+    config::Config,
+    context::Context,
+    imex::{imex, ImexMode},
+    receive_imf::receive_imf,
+    stock_str::StockStrings,
+    Events,
+};
+use tempfile::tempdir;
+
+async fn recv_all_emails(context: Context) -> Context {
+    for i in 0..100 {
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+            .await
+            .unwrap();
+    }
+    context
+}
+
+/// Receive 100 emails that remove charlie@example.com and add
+/// him back
+async fn recv_groupmembership_emails(context: Context) -> Context {
+    for i in 0..50 {
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+Chat-Group-Member-Added: charlie@example.com
+In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+            .await
+            .unwrap();
+
+        let imf_raw = format!(
+            "Subject: Benchmark
+Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Date: Sat, 07 Dec 2019 19:00:27 +0000
+To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
+From: sender@testrun.org
+Chat-Version: 1.0
+Chat-Disposition-Notification-To: sender@testrun.org
+Chat-User-Avatar: 0
+Chat-Group-Member-Removed: charlie@example.com
+In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+MIME-Version: 1.0
+
+Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
+
+Hello {i}",
+            i = i,
+            i_dec = i - 1,
+        );
+        receive_imf(&context, black_box(imf_raw.as_bytes()), false)
+            .await
+            .unwrap();
+    }
+    context
+}
+
+async fn create_context() -> Context {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let id = 100;
+    let context = Context::new(dbfile.as_path(), id, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
+
+    let backup: PathBuf = std::env::current_dir()
+        .unwrap()
+        .join("delta-chat-backup.tar");
+
+    if backup.exists() {
+        println!("Importing backup");
+        imex(&context, ImexMode::ImportBackup, backup.as_path(), None)
+            .await
+            .unwrap();
+    }
+
+    let addr = "alice@example.com";
+    context.set_config(Config::Addr, Some(addr)).await.unwrap();
+    context
+        .set_config(Config::ConfiguredAddr, Some(addr))
+        .await
+        .unwrap();
+    context
+        .set_config(Config::Configured, Some("1"))
+        .await
+        .unwrap();
+    context
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("Receive messages");
+    group.bench_function("Receive 100 simple text msgs", |b| {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let context = rt.block_on(create_context());
+
+        b.to_async(&rt).iter(|| {
+            let ctx = context.clone();
+            async move {
+                recv_all_emails(black_box(ctx)).await;
+            }
+        });
+    });
+    group.bench_function(
+        "Receive 100 Chat-Group-Member-{Added|Removed} messages",
+        |b| {
+            let rt = tokio::runtime::Runtime::new().unwrap();
+            let context = rt.block_on(create_context());
+
+            b.to_async(&rt).iter(|| {
+                let ctx = context.clone();
+                async move {
+                    recv_groupmembership_emails(black_box(ctx)).await;
+                }
+            });
+        },
+    );
+    group.finish();
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

benches/search_msgs.rs

@@ -1,12 +1,15 @@
-use async_std::task::block_on;
-use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use deltachat::context::Context;
 use std::path::Path;
 
-async fn search_benchmark(path: impl AsRef<Path>) {
-    let dbfile = path.as_ref();
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::Events;
+
+async fn search_benchmark(dbfile: impl AsRef<Path>) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.as_ref(), id, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();
@@ -17,9 +20,13 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+
         c.bench_function("search hello", |b| {
-            b.iter(|| block_on(async { search_benchmark(black_box(&path)).await }))
+            b.to_async(&rt).iter(|| search_benchmark(black_box(&path)))
         });
+    } else {
+        println!("env var not set: DELTACHAT_BENCHMARK_DATABASE");
     }
 }
 

benches/send_events.rs

@@ -0,0 +1,47 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+
+use deltachat::context::Context;
+use deltachat::stock_str::StockStrings;
+use deltachat::{info, Event, EventType, Events};
+use tempfile::tempdir;
+
+async fn send_events_benchmark(context: &Context) {
+    let emitter = context.get_event_emitter();
+    for _i in 0..1_000_000 {
+        info!(context, "interesting event...");
+    }
+    info!(context, "DONE");
+
+    loop {
+        match emitter.recv().await.unwrap() {
+            Event {
+                typ: EventType::Info(info),
+                ..
+            } if info.contains("DONE") => {
+                break;
+            }
+            _ => {}
+        }
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    let rt = tokio::runtime::Runtime::new().unwrap();
+
+    let context = rt.block_on(async {
+        Context::new(&dbfile, 100, Events::new(), StockStrings::new())
+            .await
+            .expect("failed to create context")
+    });
+    let executor = tokio::runtime::Runtime::new().unwrap();
+
+    c.bench_function("Sending 1.000.000 events", |b| {
+        b.to_async(&executor)
+            .iter(|| send_events_benchmark(&context))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

deltachat-ffi/Cargo.toml

@@ -1,8 +1,7 @@
 [package]
 name = "deltachat_ffi"
-version = "1.72.0"
+version = "1.111.0"
 description = "Deltachat FFI"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
 license = "MPL-2.0"
@@ -16,17 +15,19 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
 libc = "0.2"
 human-panic = "1"
 num-traits = "0.2"
 serde_json = "1.0"
-async-std = "1"
+tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
 rand = "0.7"
+once_cell = "1.17.0"
 
 [features]
 default = ["vendored"]
 vendored = ["deltachat/vendored"]
-nightly = ["deltachat/nightly"]
+jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/Doxyfile.css

@@ -1,14 +1,24 @@
 
+:root {
+	--accent: hsl(0 0% 85%);
+}
+
+@media (prefers-color-scheme: dark) {
+	:root {
+		--accent: hsl(0 0% 25%);
+	}
+}
+
 /* the code snippet frame, defaults to white which tends to get badly readable in combination with explaining text around */
 div.fragment {
-	background-color: #e0e0e0;
+	background-color: var(--accent);
 	border: 0;
 	padding: 1em;
 	border-radius: 6px;
 }
 
 code {
-	background-color: #e0e0e0;
+	background-color: var(--accent);
 	padding-left: .5em;
 	padding-right: .5em;
 	border-radius: 6px;

deltachat-ffi/build.rs

@@ -1,4 +1,3 @@
-use std::io::Write;
 use std::path::PathBuf;
 use std::{env, fs};
 
@@ -28,8 +27,9 @@ fn main() {
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
-    fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
-        .unwrap()
-        .write_all(pkg_config.as_bytes())
-        .unwrap();
+    fs::write(
+        target_path.join("pkgconfig").join("deltachat.pc"),
+        pkg_config.as_bytes(),
+    )
+    .unwrap();
 }

deltachat-ffi/src/dc_array.rs

@@ -1,5 +1,6 @@
 use crate::chat::ChatItem;
-use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
+use crate::constants::DC_MSG_ID_DAYMARKER;
+use crate::contact::ContactId;
 use crate::location::Location;
 use crate::message::MsgId;
 
@@ -7,6 +8,7 @@ use crate::message::MsgId;
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
     MsgIds(Vec<MsgId>),
+    ContactIds(Vec<ContactId>),
     Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
@@ -16,9 +18,9 @@ impl dc_array_t {
     pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
             Self::MsgIds(array) => array[index].to_u32(),
+            Self::ContactIds(array) => array[index].to_u32(),
             Self::Chat(array) => match array[index] {
                 ChatItem::Message { msg_id } => msg_id.to_u32(),
-                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
                 ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
             },
             Self::Locations(array) => array[index].location_id,
@@ -29,9 +31,9 @@ impl dc_array_t {
     pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
         match self {
             Self::MsgIds(_) => None,
+            Self::ContactIds(_) => None,
             Self::Chat(array) => array.get(index).and_then(|item| match item {
                 ChatItem::Message { .. } => None,
-                ChatItem::Marker1 { .. } => None,
                 ChatItem::DayMarker { timestamp } => Some(*timestamp),
             }),
             Self::Locations(array) => array.get(index).map(|location| location.timestamp),
@@ -42,6 +44,7 @@ impl dc_array_t {
     pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
         match self {
             Self::MsgIds(_) => None,
+            Self::ContactIds(_) => None,
             Self::Chat(_) => None,
             Self::Locations(array) => array
                 .get(index)
@@ -62,6 +65,7 @@ impl dc_array_t {
     pub(crate) fn len(&self) -> usize {
         match self {
             Self::MsgIds(array) => array.len(),
+            Self::ContactIds(array) => array.len(),
             Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
@@ -85,6 +89,12 @@ impl From<Vec<MsgId>> for dc_array_t {
     }
 }
 
+impl From<Vec<ContactId>> for dc_array_t {
+    fn from(array: Vec<ContactId>) -> Self {
+        dc_array_t::ContactIds(array)
+    }
+}
+
 impl From<Vec<ChatItem>> for dc_array_t {
     fn from(array: Vec<ChatItem>) -> Self {
         dc_array_t::Chat(array)

deltachat-ffi/src/lot.rs

@@ -1,10 +1,12 @@
 //! # Legacy generic return values for C API.
 
+use std::borrow::Cow;
+
+use anyhow::Error;
+
 use crate::message::MessageState;
 use crate::qr::Qr;
 use crate::summary::{Summary, SummaryPrefix};
-use anyhow::Error;
-use std::borrow::Cow;
 
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
@@ -20,20 +22,15 @@ pub enum Lot {
 }
 
 #[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub enum Meaning {
+    #[default]
     None = 0,
     Text1Draft = 1,
     Text1Username = 2,
     Text1Self = 3,
 }
 
-impl Default for Meaning {
-    fn default() -> Self {
-        Meaning::None
-    }
-}
-
 impl Lot {
     pub fn get_text1(&self) -> Option<&str> {
         match self {
@@ -51,13 +48,14 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
+                Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
                 Qr::Text { text } => Some(text),
                 Qr::WithdrawVerifyContact { .. } => None,
                 Qr::WithdrawVerifyGroup { grpname, .. } => Some(grpname),
                 Qr::ReviveVerifyContact { .. } => None,
                 Qr::ReviveVerifyGroup { grpname, .. } => Some(grpname),
+                Qr::Login { address, .. } => Some(address),
             },
             Self::Error(err) => Some(err),
         }
@@ -79,7 +77,13 @@ impl Lot {
                 Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
                 Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
             },
-            Self::Qr(_qr) => Meaning::None,
+            Self::Qr(qr) => match qr {
+                Qr::Addr {
+                    draft: Some(_draft),
+                    ..
+                } => Meaning::Text1Draft,
+                _ => Meaning::None,
+            },
             Self::Error(_err) => Meaning::None,
         }
     }
@@ -102,6 +106,7 @@ impl Lot {
                 Qr::WithdrawVerifyGroup { .. } => LotState::QrWithdrawVerifyGroup,
                 Qr::ReviveVerifyContact { .. } => LotState::QrReviveVerifyContact,
                 Qr::ReviveVerifyGroup { .. } => LotState::QrReviveVerifyGroup,
+                Qr::Login { .. } => LotState::QrLogin,
             },
             Self::Error(_err) => LotState::QrError,
         }
@@ -111,20 +116,21 @@ impl Lot {
         match self {
             Self::Summary(_) => Default::default(),
             Self::Qr(qr) => match qr {
-                Qr::AskVerifyContact { contact_id, .. } => *contact_id,
+                Qr::AskVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::AskVerifyGroup { .. } => Default::default(),
-                Qr::FprOk { contact_id } => *contact_id,
-                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default(),
+                Qr::FprOk { contact_id } => contact_id.to_u32(),
+                Qr::FprMismatch { contact_id } => contact_id.unwrap_or_default().to_u32(),
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => *contact_id,
+                Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
-                Qr::WithdrawVerifyContact { contact_id, .. } => *contact_id,
+                Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::WithdrawVerifyGroup { .. } => Default::default(),
-                Qr::ReviveVerifyContact { contact_id, .. } => *contact_id,
+                Qr::ReviveVerifyContact { contact_id, .. } => contact_id.to_u32(),
                 Qr::ReviveVerifyGroup { .. } => Default::default(),
+                Qr::Login { .. } => Default::default(),
             },
             Self::Error(_) => Default::default(),
         }
@@ -140,9 +146,9 @@ impl Lot {
 }
 
 #[repr(u32)]
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
 pub enum LotState {
-    // Default
+    #[default]
     Undefined = 0,
 
     // Qr States
@@ -189,6 +195,9 @@ pub enum LotState {
     /// text1=groupname
     QrReviveVerifyGroup = 512,
 
+    /// text1=email_address
+    QrLogin = 520,
+
     // Message States
     MsgInFresh = 10,
     MsgInNoticed = 13,
@@ -201,12 +210,6 @@ pub enum LotState {
     MsgOutMdnRcvd = 28,
 }
 
-impl Default for LotState {
-    fn default() -> Self {
-        LotState::Undefined
-    }
-}
-
 impl From<MessageState> for LotState {
     fn from(s: MessageState) -> Self {
         use MessageState::*;

deltachat-ffi/src/string.rs

@@ -55,7 +55,7 @@ pub(crate) enum CStringError {
 /// # Example
 ///
 /// ```
-/// use deltachat::dc_tools::{dc_strdup, OsStrExt};
+/// use deltachat::tools::{dc_strdup, OsStrExt};
 /// let path = std::path::Path::new("/some/path");
 /// let path_c = path.to_c_string().unwrap();
 /// unsafe {
@@ -287,9 +287,10 @@ fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
     use libc::{free, strcmp};
 
+    use super::*;
+
     #[test]
     fn test_os_str_to_c_string_cwd() {
         let some_dir = std::env::current_dir().unwrap();

deltachat-jsonrpc/.gitignore

@@ -0,0 +1,3 @@
+accounts/
+
+.cargo

deltachat-jsonrpc/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "deltachat-jsonrpc"
+version = "1.111.0"
+description = "DeltaChat JSON-RPC API"
+edition = "2021"
+default-run = "deltachat-jsonrpc-server"
+license = "MPL-2.0"
+
+[[bin]]
+name = "deltachat-jsonrpc-server"
+path = "src/webserver.rs"
+required-features = ["webserver"]
+
+[dependencies]
+anyhow = "1"
+deltachat = { path = ".." }
+num-traits = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+tempfile = "3.3.0"
+log = "0.4"
+async-channel = { version = "1.8.0" }
+futures = { version = "0.3.26" }
+serde_json = "1.0.91"
+yerpc = { version = "0.4.3", features = ["anyhow_expose"] }
+typescript-type-def = { version = "0.5.5", features = ["json_value"] }
+tokio = { version = "1.25.0" }
+sanitize-filename = "0.4"
+walkdir = "2.3.2"
+base64 = "0.21"
+
+# optional dependencies
+axum = { version = "0.6.6", optional = true, features = ["ws"] }
+env_logger = { version = "0.10.0", optional = true }
+
+[dev-dependencies]
+tokio = { version = "1.25.0", features = ["full", "rt-multi-thread"] }
+
+
+[features]
+default = ["vendored"]
+webserver = ["dep:env_logger", "dep:axum", "tokio/full", "yerpc/support-axum"]
+vendored = ["deltachat/vendored"]

deltachat-jsonrpc/README.md

@@ -0,0 +1,123 @@
+# deltachat-jsonrpc
+
+This crate provides a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) interface to DeltaChat.
+
+The JSON-RPC API is exposed in two fashions:
+
+* A executable that exposes the JSON-RPC API through a [WebSocket](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API) server running on localhost.
+* The JSON-RPC API can also be called through the [C FFI](../deltachat-ffi). The C FFI needs to be built with the `jsonrpc` feature. It will then expose the functions `dc_jsonrpc_init`, `dc_jsonrpc_request`, `dc_jsonrpc_next_response` and `dc_jsonrpc_unref`. See the docs in the [header file](../deltachat-ffi/deltachat.h) for details.
+
+We also include a JavaScript and TypeScript client for the JSON-RPC API. The source for this is in the [`typescript`](typescript) folder. The client can easily be used with the WebSocket server to build DeltaChat apps for web browsers or Node.js. See the [examples](typescript/example) for details.
+
+## Usage
+
+#### Running the WebSocket server
+
+From within this folder, you can start the WebSocket server with the following command:
+
+```sh
+cargo run --features webserver
+```
+
+If you want to use the server in a production setup, first build it in release mode:
+
+```sh
+cargo build --features webserver --release
+```
+You will then find the `deltachat-jsonrpc-server` executable in your `target/release` folder.
+
+The executable currently does not support any command-line arguments. By default, once started it will accept WebSocket connections on `ws://localhost:20808/ws`. It will store the persistent configuration and databases in a `./accounts` folder relative to the directory from where it is started.
+
+The server can be configured with environment variables:
+
+|variable|default|description|
+|-|-|-|
+|`DC_PORT`|`20808`|port to listen on|
+|`DC_ACCOUNTS_PATH`|`./accounts`|path to storage directory|
+
+If you are targeting other architectures (like KaiOS or Android), the webserver binary can be cross-compiled easily with [rust-cross](https://github.com/cross-rs/cross):
+
+```sh
+cross build --features=webserver --target armv7-linux-androideabi --release
+```
+
+#### Using the TypeScript/JavaScript client
+
+The package includes a JavaScript/TypeScript client which is partially auto-generated through the JSON-RPC library used by this crate ([yerpc](https://github.com/Frando/yerpc/)). Find the source in the [`typescript`](typescript) folder. 
+
+To use it locally, first install the dependencies and compile the TypeScript code to JavaScript:
+```sh
+cd typescript
+npm install
+npm run build
+```
+
+The JavaScript client is not yet published on NPM (but will likely be soon). Currently, it is recommended to vendor the bundled build. After running `npm run build` as documented above, there will be a file `dist/deltachat.bundle.js`. This is an ESM module containing all dependencies. Copy this file to your project and import the DeltaChat class.
+
+
+```typescript
+import { DeltaChat } from './deltachat.bundle.js'
+const dc = new DeltaChat('ws://localhost:20808/ws')
+const accounts = await dc.rpc.getAllAccounts()
+console.log('accounts', accounts)
+```
+
+A script is included to build autogenerated documentation, which includes all RPC methods:
+```sh
+cd typescript
+npm run docs
+```
+Then open the [`typescript/docs`](typescript/docs) folder in a web browser.
+
+## Development
+
+#### Running the example app
+
+We include a small demo web application that talks to the WebSocket server. It can be used for testing. Feel invited to expand this.
+
+```sh
+cd typescript
+npm run build
+npm run example:build
+npm run example:start
+```
+Then, open [`http://localhost:8080/example.html`](http://localhost:8080/example.html) in a web browser.
+
+Run `npm run example:dev` to live-rebuild the example app when files changes.
+
+### Testing
+
+The crate includes both a basic Rust smoke test and more featureful integration tests that use the TypeScript client.
+
+#### Rust tests
+
+To run the Rust test, use this command:
+
+```
+cargo test
+```
+
+#### TypeScript tests
+
+```
+cd typescript
+npm run test
+```
+
+This will build the `deltachat-jsonrpc-server` binary and then run a test suite against the WebSocket server.
+
+The test suite includes some tests that need online connectivity and a way to create test email accounts. To run these tests, talk to DeltaChat developers to get a token for the `testrun.org` service, or use a local instance of [`mailadm`](https://github.com/deltachat/docker-mailadm).
+
+Then, set the `DCC_NEW_TMP_EMAIL` environment variable to your mailadm token before running the tests.
+
+```
+DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=yourtoken npm run test
+```
+
+#### Test Coverage
+
+Running `npm run test` will report test coverage. For the coverage to be accurate the online tests need to be run.
+
+> If you are offline and want to see the coverage results anyway (even though they are inaccurate), you can bypass the errors of the online tests by setting the `COVERAGE_OFFLINE=1` environment variable.
+
+A summary of the coverage will be reported in the terminal after the test run. Open `coverage/index.html` in a web browser for a detailed report.

deltachat-jsonrpc/TODO.md

@@ -0,0 +1,28 @@
+# TODO
+
+- [ ] different test type to simulate two devices: to test autocrypt_initiate_key_transfer & autocrypt_continue_key_transfer
+
+## MVP - Websocket server&client
+
+For kaiOS and other experiments, like a deltachat "web" over network from an android phone.
+
+- [ ] coverage for a majority of the API
+- [ ] Blobs served
+- [ ] Blob upload (for attachments, setting profile-picture, importing backup and so on)
+- [ ] other way blobs can be addressed when using websocket vs. jsonrpc over dc-node
+- [ ] Web push API? At least some kind of notification hook closure this lib can accept.
+
+### Other Ideas for the Websocket server
+
+- [ ] make sure there can only be one connection at a time to the ws
+  - why? , it could give problems if its commanded from multiple connections
+- [ ] encrypted connection?
+- [ ] authenticated connection?
+- [ ] Look into unit-testing for the proc macros?
+- [ ] proc macro taking over doc comments to generated typescript file
+
+## Desktop Apis
+
+Incomplete todo for desktop api porting, just some remainders for points that might need more work:
+
+- [ ] manual start/stop io functions in the api for context and accounts, so "not syncing all accounts" can still be done in desktop -> webserver should then not do start io on all accounts by default

deltachat-jsonrpc/src/api/events.rs

@@ -0,0 +1,402 @@
+use deltachat::{Event, EventType};
+use serde::Serialize;
+use serde_json::{json, Value};
+use typescript_type_def::TypeDef;
+
+pub fn event_to_json_rpc_notification(event: Event) -> Value {
+    let id: JSONRPCEventType = event.typ.into();
+    json!({
+        "event": id,
+        "contextId": event.id,
+    })
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type", rename = "Event")]
+pub enum JSONRPCEventType {
+    /// The library-user may write an informational string to the log.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
+    Info {
+        msg: String,
+    },
+
+    /// Emitted when SMTP connection is established and login was successful.
+    SmtpConnected {
+        msg: String,
+    },
+
+    /// Emitted when IMAP connection is established and login was successful.
+    ImapConnected {
+        msg: String,
+    },
+
+    /// Emitted when a message was successfully sent to the SMTP server.
+    SmtpMessageSent {
+        msg: String,
+    },
+
+    /// Emitted when an IMAP message has been marked as deleted
+    ImapMessageDeleted {
+        msg: String,
+    },
+
+    /// Emitted when an IMAP message has been moved
+    ImapMessageMoved {
+        msg: String,
+    },
+
+    /// Emitted when an new file in the $BLOBDIR was created
+    NewBlobFile {
+        file: String,
+    },
+
+    /// Emitted when an file in the $BLOBDIR was deleted
+    DeletedBlobFile {
+        file: String,
+    },
+
+    /// The library-user should write a warning string to the log.
+    ///
+    /// This event should *not* be reported to the end-user using a popup or something like
+    /// that.
+    Warning {
+        msg: String,
+    },
+
+    /// The library-user should report an error to the end-user.
+    ///
+    /// As most things are asynchronous, things may go wrong at any time and the user
+    /// should not be disturbed by a dialog or so.  Instead, use a bubble or so.
+    ///
+    /// However, for ongoing processes (eg. configure())
+    /// or for functions that are expected to fail (eg. autocryptContinueKeyTransfer())
+    /// it might be better to delay showing these events until the function has really
+    /// failed (returned false). It should be sufficient to report only the *last* error
+    /// in a messasge box then.
+    Error {
+        msg: String,
+    },
+
+    /// An action cannot be performed because the user is not in the group.
+    /// Reported eg. after a call to
+    /// setChatName(), setChatProfileImage(),
+    /// addContactToChat(), removeContactFromChat(),
+    /// and messages sending functions.
+    ErrorSelfNotInGroup {
+        msg: String,
+    },
+
+    /// Messages or chats changed.  One or more messages or chats changed for various
+    /// reasons in the database:
+    /// - Messages sent, received or removed
+    /// - Chats created, deleted or archived
+    /// - A draft has been set
+    ///
+    /// `chatId` is set if only a single chat is affected by the changes, otherwise 0.
+    /// `msgId` is set if only a single message is affected by the changes, otherwise 0.
+    #[serde(rename_all = "camelCase")]
+    MsgsChanged {
+        chat_id: u32,
+        msg_id: u32,
+    },
+
+    /// Reactions for the message changed.
+    #[serde(rename_all = "camelCase")]
+    ReactionsChanged {
+        chat_id: u32,
+        msg_id: u32,
+        contact_id: u32,
+    },
+
+    /// There is a fresh message. Typically, the user will show an notification
+    /// when receiving this message.
+    ///
+    /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
+    #[serde(rename_all = "camelCase")]
+    IncomingMsg {
+        chat_id: u32,
+        msg_id: u32,
+    },
+
+    /// Downloading a bunch of messages just finished. This is an experimental
+    /// event to allow the UI to only show one notification per message bunch,
+    /// instead of cluttering the user with many notifications.
+    ///
+    /// msg_ids contains the message ids.
+    #[serde(rename_all = "camelCase")]
+    IncomingMsgBunch {
+        msg_ids: Vec<u32>,
+    },
+
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[serde(rename_all = "camelCase")]
+    MsgsNoticed {
+        chat_id: u32,
+    },
+
+    /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
+    /// DC_STATE_OUT_DELIVERED, see `Message.state`.
+    #[serde(rename_all = "camelCase")]
+    MsgDelivered {
+        chat_id: u32,
+        msg_id: u32,
+    },
+
+    /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
+    /// DC_STATE_OUT_FAILED, see `Message.state`.
+    #[serde(rename_all = "camelCase")]
+    MsgFailed {
+        chat_id: u32,
+        msg_id: u32,
+    },
+
+    /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
+    /// DC_STATE_OUT_MDN_RCVD, see `Message.state`.
+    #[serde(rename_all = "camelCase")]
+    MsgRead {
+        chat_id: u32,
+        msg_id: u32,
+    },
+
+    /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
+    /// Or the verify state of a chat has changed.
+    /// See setChatName(), setChatProfileImage(), addContactToChat()
+    /// and removeContactFromChat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
+    #[serde(rename_all = "camelCase")]
+    ChatModified {
+        chat_id: u32,
+    },
+
+    /// Chat ephemeral timer changed.
+    #[serde(rename_all = "camelCase")]
+    ChatEphemeralTimerModified {
+        chat_id: u32,
+        timer: u32,
+    },
+
+    /// Contact(s) created, renamed, blocked or deleted.
+    ///
+    /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
+    #[serde(rename_all = "camelCase")]
+    ContactsChanged {
+        contact_id: Option<u32>,
+    },
+
+    /// Location of one or more contact has changed.
+    ///
+    /// @param data1 (u32) contact_id of the contact for which the location has changed.
+    ///     If the locations of several contacts have been changed,
+    ///     this parameter is set to `None`.
+    #[serde(rename_all = "camelCase")]
+    LocationChanged {
+        contact_id: Option<u32>,
+    },
+
+    /// Inform about the configuration progress started by configure().
+    ConfigureProgress {
+        /// Progress.
+        ///
+        /// 0=error, 1-999=progress in permille, 1000=success and done
+        progress: usize,
+
+        /// Progress comment or error, something to display to the user.
+        comment: Option<String>,
+    },
+
+    /// Inform about the import/export progress started by imex().
+    ///
+    /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
+    /// @param data2 0
+    #[serde(rename_all = "camelCase")]
+    ImexProgress {
+        progress: usize,
+    },
+
+    /// A file has been exported. A file has been written by imex().
+    /// This event may be sent multiple times by a single call to imex().
+    ///
+    /// A typical purpose for a handler of this event may be to make the file public to some system
+    /// services.
+    ///
+    /// @param data2 0
+    #[serde(rename_all = "camelCase")]
+    ImexFileWritten {
+        path: String,
+    },
+
+    /// Progress information of a secure-join handshake from the view of the inviter
+    /// (Alice, the person who shows the QR code).
+    ///
+    /// These events are typically sent after a joiner has scanned the QR code
+    /// generated by getChatSecurejoinQrCodeSvg().
+    ///
+    /// @param data1 (int) ID of the contact that wants to join.
+    /// @param data2 (int) Progress as:
+    ///     300=vg-/vc-request received, typically shown as "bob@addr joins".
+    ///     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
+    ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
+    ///     1000=Protocol finished for this contact.
+    #[serde(rename_all = "camelCase")]
+    SecurejoinInviterProgress {
+        contact_id: u32,
+        progress: usize,
+    },
+
+    /// Progress information of a secure-join handshake from the view of the joiner
+    /// (Bob, the person who scans the QR code).
+    /// The events are typically sent while secureJoin(), which
+    /// may take some time, is executed.
+    /// @param data1 (int) ID of the inviting contact.
+    /// @param data2 (int) Progress as:
+    ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
+    ///     (Bob has verified alice and waits until Alice does the same for him)
+    #[serde(rename_all = "camelCase")]
+    SecurejoinJoinerProgress {
+        contact_id: u32,
+        progress: usize,
+    },
+
+    /// The connectivity to the server changed.
+    /// This means that you should refresh the connectivity view
+    /// and possibly the connectivtiy HTML; see getConnectivity() and
+    /// getConnectivityHtml() for details.
+    ConnectivityChanged,
+
+    SelfavatarChanged,
+
+    #[serde(rename_all = "camelCase")]
+    WebxdcStatusUpdate {
+        msg_id: u32,
+        status_update_serial: u32,
+    },
+
+    /// Inform that a message containing a webxdc instance has been deleted
+    #[serde(rename_all = "camelCase")]
+    WebxdcInstanceDeleted {
+        msg_id: u32,
+    },
+}
+
+impl From<EventType> for JSONRPCEventType {
+    fn from(event: EventType) -> Self {
+        use JSONRPCEventType::*;
+        match event {
+            EventType::Info(msg) => Info { msg },
+            EventType::SmtpConnected(msg) => SmtpConnected { msg },
+            EventType::ImapConnected(msg) => ImapConnected { msg },
+            EventType::SmtpMessageSent(msg) => SmtpMessageSent { msg },
+            EventType::ImapMessageDeleted(msg) => ImapMessageDeleted { msg },
+            EventType::ImapMessageMoved(msg) => ImapMessageMoved { msg },
+            EventType::NewBlobFile(file) => NewBlobFile { file },
+            EventType::DeletedBlobFile(file) => DeletedBlobFile { file },
+            EventType::Warning(msg) => Warning { msg },
+            EventType::Error(msg) => Error { msg },
+            EventType::ErrorSelfNotInGroup(msg) => ErrorSelfNotInGroup { msg },
+            EventType::MsgsChanged { chat_id, msg_id } => MsgsChanged {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+            },
+            EventType::ReactionsChanged {
+                chat_id,
+                msg_id,
+                contact_id,
+            } => ReactionsChanged {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+                contact_id: contact_id.to_u32(),
+            },
+            EventType::IncomingMsg { chat_id, msg_id } => IncomingMsg {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+            },
+            EventType::IncomingMsgBunch { msg_ids } => IncomingMsgBunch {
+                msg_ids: msg_ids.into_iter().map(|id| id.to_u32()).collect(),
+            },
+            EventType::MsgsNoticed(chat_id) => MsgsNoticed {
+                chat_id: chat_id.to_u32(),
+            },
+            EventType::MsgDelivered { chat_id, msg_id } => MsgDelivered {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+            },
+            EventType::MsgFailed { chat_id, msg_id } => MsgFailed {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+            },
+            EventType::MsgRead { chat_id, msg_id } => MsgRead {
+                chat_id: chat_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+            },
+            EventType::ChatModified(chat_id) => ChatModified {
+                chat_id: chat_id.to_u32(),
+            },
+            EventType::ChatEphemeralTimerModified { chat_id, timer } => {
+                ChatEphemeralTimerModified {
+                    chat_id: chat_id.to_u32(),
+                    timer: timer.to_u32(),
+                }
+            }
+            EventType::ContactsChanged(contact) => ContactsChanged {
+                contact_id: contact.map(|c| c.to_u32()),
+            },
+            EventType::LocationChanged(contact) => LocationChanged {
+                contact_id: contact.map(|c| c.to_u32()),
+            },
+            EventType::ConfigureProgress { progress, comment } => {
+                ConfigureProgress { progress, comment }
+            }
+            EventType::ImexProgress(progress) => ImexProgress { progress },
+            EventType::ImexFileWritten(path) => ImexFileWritten {
+                path: path.to_str().unwrap_or_default().to_owned(),
+            },
+            EventType::SecurejoinInviterProgress {
+                contact_id,
+                progress,
+            } => SecurejoinInviterProgress {
+                contact_id: contact_id.to_u32(),
+                progress,
+            },
+            EventType::SecurejoinJoinerProgress {
+                contact_id,
+                progress,
+            } => SecurejoinJoinerProgress {
+                contact_id: contact_id.to_u32(),
+                progress,
+            },
+            EventType::ConnectivityChanged => ConnectivityChanged,
+            EventType::SelfavatarChanged => SelfavatarChanged,
+            EventType::WebxdcStatusUpdate {
+                msg_id,
+                status_update_serial,
+            } => WebxdcStatusUpdate {
+                msg_id: msg_id.to_u32(),
+                status_update_serial: status_update_serial.to_u32(),
+            },
+            EventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
+                msg_id: msg_id.to_u32(),
+            },
+        }
+    }
+}
+
+#[cfg(test)]
+#[test]
+fn generate_events_ts_types_definition() {
+    let events = {
+        let mut buf = Vec::new();
+        let options = typescript_type_def::DefinitionFileOptions {
+            root_namespace: None,
+            ..typescript_type_def::DefinitionFileOptions::default()
+        };
+        typescript_type_def::write_definition_file::<_, JSONRPCEventType>(&mut buf, options)
+            .unwrap();
+        String::from_utf8(buf).unwrap()
+    };
+    std::fs::write("typescript/generated/events.ts", events).unwrap();
+}

deltachat-jsonrpc/src/api/types/account.rs

@@ -0,0 +1,45 @@
+use anyhow::Result;
+use deltachat::config::Config;
+use deltachat::contact::{Contact, ContactId};
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum Account {
+    #[serde(rename_all = "camelCase")]
+    Configured {
+        id: u32,
+        display_name: Option<String>,
+        addr: Option<String>,
+        // size: u32,
+        profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
+        color: String,
+    },
+    #[serde(rename_all = "camelCase")]
+    Unconfigured { id: u32 },
+}
+
+impl Account {
+    pub async fn from_context(ctx: &deltachat::context::Context, id: u32) -> Result<Self> {
+        if ctx.is_configured().await? {
+            let display_name = ctx.get_config(Config::Displayname).await?;
+            let addr = ctx.get_config(Config::Addr).await?;
+            let profile_image = ctx.get_config(Config::Selfavatar).await?;
+            let color = color_int_to_hex_string(
+                Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
+            );
+            Ok(Account::Configured {
+                id,
+                display_name,
+                addr,
+                profile_image,
+                color,
+            })
+        } else {
+            Ok(Account::Unconfigured { id })
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/chat.rs

@@ -0,0 +1,213 @@
+use std::time::{Duration, SystemTime};
+
+use anyhow::{anyhow, bail, Result};
+use deltachat::chat::{self, get_chat_contacts, ChatVisibility};
+use deltachat::chat::{Chat, ChatId};
+use deltachat::constants::Chattype;
+use deltachat::contact::{Contact, ContactId};
+use deltachat::context::Context;
+use num_traits::cast::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+use super::contact::ContactObject;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct FullChat {
+    id: u32,
+    name: String,
+    is_protected: bool,
+    profile_image: Option<String>, //BLOBS ?
+    archived: bool,
+    // subtitle  - will be moved to frontend because it uses translation functions
+    chat_type: u32,
+    is_unpromoted: bool,
+    is_self_talk: bool,
+    contacts: Vec<ContactObject>,
+    contact_ids: Vec<u32>,
+    color: String,
+    fresh_message_counter: usize,
+    // is_group - please check over chat.type in frontend instead
+    is_contact_request: bool,
+    is_device_chat: bool,
+    self_in_group: bool,
+    is_muted: bool,
+    ephemeral_timer: u32, //TODO look if there are more important properties in newer core versions
+    can_send: bool,
+    was_seen_recently: bool,
+    mailing_list_address: Option<String>,
+}
+
+impl FullChat {
+    pub async fn try_from_dc_chat_id(context: &Context, chat_id: u32) -> Result<Self> {
+        let rust_chat_id = ChatId::new(chat_id);
+        let chat = Chat::load_from_db(context, rust_chat_id).await?;
+
+        let contact_ids = get_chat_contacts(context, rust_chat_id).await?;
+
+        let mut contacts = Vec::with_capacity(contact_ids.len());
+
+        for contact_id in &contact_ids {
+            contacts.push(
+                ContactObject::try_from_dc_contact(
+                    context,
+                    Contact::load_from_db(context, *contact_id).await?,
+                )
+                .await?,
+            )
+        }
+
+        let profile_image = match chat.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+
+        let color = color_int_to_hex_string(chat.get_color(context).await?);
+        let fresh_message_counter = rust_chat_id.get_fresh_msg_cnt(context).await?;
+        let ephemeral_timer = rust_chat_id.get_ephemeral_timer(context).await?.to_u32();
+
+        let can_send = chat.can_send(context).await?;
+
+        let was_seen_recently = if chat.get_type() == Chattype::Single {
+            match contact_ids.get(0) {
+                Some(contact) => Contact::load_from_db(context, *contact)
+                    .await?
+                    .was_seen_recently(),
+                None => false,
+            }
+        } else {
+            false
+        };
+
+        let mailing_list_address = chat.get_mailinglist_addr().map(|s| s.to_string());
+
+        Ok(FullChat {
+            id: chat_id,
+            name: chat.name.clone(),
+            is_protected: chat.is_protected(),
+            profile_image, //BLOBS ?
+            archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            chat_type: chat
+                .get_type()
+                .to_u32()
+                .ok_or_else(|| anyhow!("unknown chat type id"))?, // TODO get rid of this unwrap?
+            is_unpromoted: chat.is_unpromoted(),
+            is_self_talk: chat.is_self_talk(),
+            contacts,
+            contact_ids: contact_ids.iter().map(|id| id.to_u32()).collect(),
+            color,
+            fresh_message_counter,
+            is_contact_request: chat.is_contact_request(),
+            is_device_chat: chat.is_device_talk(),
+            self_in_group: contact_ids.contains(&ContactId::SELF),
+            is_muted: chat.is_muted(),
+            ephemeral_timer,
+            can_send,
+            was_seen_recently,
+            mailing_list_address,
+        })
+    }
+}
+
+/// cheaper version of fullchat, omits:
+/// - contacts
+/// - contact_ids
+/// - fresh_message_counter
+/// - ephemeral_timer
+/// - self_in_group
+/// - was_seen_recently
+/// - can_send
+///
+/// used when you only need the basic metadata of a chat like type, name, profile picture
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct BasicChat {
+    id: u32,
+    name: String,
+    is_protected: bool,
+    profile_image: Option<String>, //BLOBS ?
+    archived: bool,
+    chat_type: u32,
+    is_unpromoted: bool,
+    is_self_talk: bool,
+    color: String,
+    is_contact_request: bool,
+    is_device_chat: bool,
+    is_muted: bool,
+}
+
+impl BasicChat {
+    pub async fn try_from_dc_chat_id(context: &Context, chat_id: u32) -> Result<Self> {
+        let rust_chat_id = ChatId::new(chat_id);
+        let chat = Chat::load_from_db(context, rust_chat_id).await?;
+
+        let profile_image = match chat.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+        let color = color_int_to_hex_string(chat.get_color(context).await?);
+
+        Ok(BasicChat {
+            id: chat_id,
+            name: chat.name.clone(),
+            is_protected: chat.is_protected(),
+            profile_image, //BLOBS ?
+            archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            chat_type: chat
+                .get_type()
+                .to_u32()
+                .ok_or_else(|| anyhow!("unknown chat type id"))?, // TODO get rid of this unwrap?
+            is_unpromoted: chat.is_unpromoted(),
+            is_self_talk: chat.is_self_talk(),
+            color,
+            is_contact_request: chat.is_contact_request(),
+            is_device_chat: chat.is_device_talk(),
+            is_muted: chat.is_muted(),
+        })
+    }
+}
+
+#[derive(Clone, Serialize, Deserialize, TypeDef)]
+pub enum MuteDuration {
+    NotMuted,
+    Forever,
+    Until(i64),
+}
+
+impl MuteDuration {
+    pub fn try_into_core_type(self) -> Result<chat::MuteDuration> {
+        match self {
+            MuteDuration::NotMuted => Ok(chat::MuteDuration::NotMuted),
+            MuteDuration::Forever => Ok(chat::MuteDuration::Forever),
+            MuteDuration::Until(n) => {
+                if n <= 0 {
+                    bail!("failed to read mute duration")
+                }
+
+                Ok(SystemTime::now()
+                    .checked_add(Duration::from_secs(n as u64))
+                    .map_or(chat::MuteDuration::Forever, chat::MuteDuration::Until))
+            }
+        }
+    }
+}
+
+#[derive(Clone, Serialize, Deserialize, TypeDef)]
+#[serde(rename = "ChatVisibility")]
+pub enum JSONRPCChatVisibility {
+    Normal,
+    Archived,
+    Pinned,
+}
+
+impl JSONRPCChatVisibility {
+    pub fn into_core_type(self) -> ChatVisibility {
+        match self {
+            JSONRPCChatVisibility::Normal => ChatVisibility::Normal,
+            JSONRPCChatVisibility::Archived => ChatVisibility::Archived,
+            JSONRPCChatVisibility::Pinned => ChatVisibility::Pinned,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -0,0 +1,142 @@
+use anyhow::Result;
+use deltachat::constants::*;
+use deltachat::contact::{Contact, ContactId};
+use deltachat::{
+    chat::{get_chat_contacts, ChatVisibility},
+    chatlist::Chatlist,
+};
+use deltachat::{
+    chat::{Chat, ChatId},
+    message::MsgId,
+};
+use num_traits::cast::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Deserialize, Serialize, TypeDef)]
+pub struct ChatListEntry(pub u32, pub u32);
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum ChatListItemFetchResult {
+    #[serde(rename_all = "camelCase")]
+    ChatListItem {
+        id: u32,
+        name: String,
+        avatar_path: Option<String>,
+        color: String,
+        last_updated: Option<i64>,
+        summary_text1: String,
+        summary_text2: String,
+        summary_status: u32,
+        is_protected: bool,
+        is_group: bool,
+        fresh_message_counter: usize,
+        is_self_talk: bool,
+        is_device_talk: bool,
+        is_sending_location: bool,
+        is_self_in_group: bool,
+        is_archived: bool,
+        is_pinned: bool,
+        is_muted: bool,
+        is_contact_request: bool,
+        /// true when chat is a broadcastlist
+        is_broadcast: bool,
+        /// contact id if this is a dm chat (for view profile entry in context menu)
+        dm_chat_contact: Option<u32>,
+        was_seen_recently: bool,
+    },
+    #[serde(rename_all = "camelCase")]
+    ArchiveLink { fresh_message_counter: usize },
+    #[serde(rename_all = "camelCase")]
+    Error { id: u32, error: String },
+}
+
+pub(crate) async fn get_chat_list_item_by_id(
+    ctx: &deltachat::context::Context,
+    entry: &ChatListEntry,
+) -> Result<ChatListItemFetchResult> {
+    let chat_id = ChatId::new(entry.0);
+    let last_msgid = match entry.1 {
+        0 => None,
+        _ => Some(MsgId::new(entry.1)),
+    };
+
+    let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?;
+
+    if chat_id.is_archived_link() {
+        return Ok(ChatListItemFetchResult::ArchiveLink {
+            fresh_message_counter,
+        });
+    }
+
+    let chat = Chat::load_from_db(ctx, chat_id).await?;
+    let summary = Chatlist::get_summary2(ctx, chat_id, last_msgid, Some(&chat)).await?;
+
+    let summary_text1 = summary.prefix.map_or_else(String::new, |s| s.to_string());
+    let summary_text2 = summary.text.to_owned();
+
+    let visibility = chat.get_visibility();
+
+    let avatar_path = chat
+        .get_profile_image(ctx)
+        .await?
+        .map(|path| path.to_str().unwrap_or("invalid/path").to_owned());
+
+    let last_updated = match last_msgid {
+        Some(id) => {
+            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
+            Some(last_message.get_timestamp() * 1000)
+        }
+        None => None,
+    };
+
+    let chat_contacts = get_chat_contacts(ctx, chat_id).await?;
+
+    let self_in_group = chat_contacts.contains(&ContactId::SELF);
+
+    let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
+        let contact = chat_contacts.get(0);
+        let was_seen_recently = match contact {
+            Some(contact) => Contact::load_from_db(ctx, *contact)
+                .await?
+                .was_seen_recently(),
+            None => false,
+        };
+        (
+            contact.map(|contact_id| contact_id.to_u32()),
+            was_seen_recently,
+        )
+    } else {
+        (None, false)
+    };
+
+    let color = color_int_to_hex_string(chat.get_color(ctx).await?);
+
+    Ok(ChatListItemFetchResult::ChatListItem {
+        id: chat_id.to_u32(),
+        name: chat.get_name().to_owned(),
+        avatar_path,
+        color,
+        last_updated,
+        summary_text1,
+        summary_text2,
+        summary_status: summary.state.to_u32().expect("impossible"), // idea and a function to transform the constant to strings? or return string enum
+        is_protected: chat.is_protected(),
+        is_group: chat.get_type() == Chattype::Group,
+        fresh_message_counter,
+        is_self_talk: chat.is_self_talk(),
+        is_device_talk: chat.is_device_talk(),
+        is_self_in_group: self_in_group,
+        is_sending_location: chat.is_sending_locations(),
+        is_archived: visibility == ChatVisibility::Archived,
+        is_pinned: visibility == ChatVisibility::Pinned,
+        is_muted: chat.is_muted(),
+        is_contact_request: chat.is_contact_request(),
+        is_broadcast: chat.get_type() == Chattype::Broadcast,
+        dm_chat_contact,
+        was_seen_recently,
+    })
+}

deltachat-jsonrpc/src/api/types/contact.rs

@@ -0,0 +1,73 @@
+use anyhow::Result;
+use deltachat::contact::VerifiedStatus;
+use deltachat::context::Context;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Contact", rename_all = "camelCase")]
+pub struct ContactObject {
+    address: String,
+    color: String,
+    auth_name: String,
+    status: String,
+    display_name: String,
+    id: u32,
+    name: String,
+    profile_image: Option<String>, // BLOBS
+    name_and_addr: String,
+    is_blocked: bool,
+    is_verified: bool,
+    /// the address that verified this contact
+    verifier_addr: Option<String>,
+    /// the id of the contact that verified this contact
+    verifier_id: Option<u32>,
+    /// the contact's last seen timestamp
+    last_seen: i64,
+    was_seen_recently: bool,
+}
+
+impl ContactObject {
+    pub async fn try_from_dc_contact(
+        context: &Context,
+        contact: deltachat::contact::Contact,
+    ) -> Result<Self> {
+        let profile_image = match contact.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+
+        let (verifier_addr, verifier_id) = if is_verified {
+            (
+                contact.get_verifier_addr(context).await?,
+                contact
+                    .get_verifier_id(context)
+                    .await?
+                    .map(|contact_id| contact_id.to_u32()),
+            )
+        } else {
+            (None, None)
+        };
+
+        Ok(ContactObject {
+            address: contact.get_addr().to_owned(),
+            color: color_int_to_hex_string(contact.get_color()),
+            auth_name: contact.get_authname().to_owned(),
+            status: contact.get_status().to_owned(),
+            display_name: contact.get_display_name().to_owned(),
+            id: contact.id.to_u32(),
+            name: contact.get_name().to_owned(),
+            profile_image, //BLOBS
+            name_and_addr: contact.get_name_n_addr(),
+            is_blocked: contact.is_blocked(),
+            is_verified,
+            verifier_addr,
+            verifier_id,
+            last_seen: contact.last_seen(),
+            was_seen_recently: contact.was_seen_recently(),
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/location.rs

@@ -0,0 +1,47 @@
+use deltachat::location::Location;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Location", rename_all = "camelCase")]
+pub struct JsonrpcLocation {
+    pub location_id: u32,
+    pub is_independent: bool,
+    pub latitude: f64,
+    pub longitude: f64,
+    pub accuracy: f64,
+    pub timestamp: i64,
+    pub contact_id: u32,
+    pub msg_id: u32,
+    pub chat_id: u32,
+    pub marker: Option<String>,
+}
+
+impl From<Location> for JsonrpcLocation {
+    fn from(location: Location) -> Self {
+        let Location {
+            location_id,
+            independent,
+            latitude,
+            longitude,
+            accuracy,
+            timestamp,
+            contact_id,
+            msg_id,
+            chat_id,
+            marker,
+        } = location;
+        Self {
+            location_id,
+            is_independent: independent != 0,
+            latitude,
+            longitude,
+            accuracy,
+            timestamp,
+            contact_id: contact_id.to_u32(),
+            msg_id,
+            chat_id: chat_id.to_u32(),
+            marker,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/message.rs

@@ -0,0 +1,516 @@
+use anyhow::{anyhow, Result};
+use deltachat::chat::Chat;
+use deltachat::chat::ChatItem;
+use deltachat::constants::Chattype;
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use deltachat::download;
+use deltachat::message::Message;
+use deltachat::message::MsgId;
+use deltachat::message::Viewtype;
+use deltachat::reaction::get_msg_reactions;
+use num_traits::cast::ToPrimitive;
+use serde::Deserialize;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+use super::contact::ContactObject;
+use super::reactions::JSONRPCReactions;
+use super::webxdc::WebxdcMessageInfo;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase", tag = "variant")]
+pub enum MessageLoadResult {
+    Message(MessageObject),
+    LoadingError { error: String },
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Message", rename_all = "camelCase")]
+pub struct MessageObject {
+    id: u32,
+    chat_id: u32,
+    from_id: u32,
+    quote: Option<MessageQuote>,
+    parent_id: Option<u32>,
+
+    text: Option<String>,
+    has_location: bool,
+    has_html: bool,
+    view_type: MessageViewtype,
+    state: u32,
+
+    /// An error text, if there is one.
+    error: Option<String>,
+
+    timestamp: i64,
+    sort_timestamp: i64,
+    received_timestamp: i64,
+    has_deviating_timestamp: bool,
+
+    // summary - use/create another function if you need it
+    subject: String,
+    show_padlock: bool,
+    is_setupmessage: bool,
+    is_info: bool,
+    is_forwarded: bool,
+
+    /// True if the message was sent by a bot.
+    is_bot: bool,
+
+    /// when is_info is true this describes what type of system message it is
+    system_message_type: SystemMessageType,
+
+    duration: i32,
+    dimensions_height: i32,
+    dimensions_width: i32,
+
+    videochat_type: Option<u32>,
+    videochat_url: Option<String>,
+
+    override_sender_name: Option<String>,
+    sender: ContactObject,
+
+    setup_code_begin: Option<String>,
+
+    file: Option<String>,
+    file_mime: Option<String>,
+    file_bytes: u64,
+    file_name: Option<String>,
+
+    webxdc_info: Option<WebxdcMessageInfo>,
+
+    download_state: DownloadState,
+
+    reactions: Option<JSONRPCReactions>,
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "kind")]
+enum MessageQuote {
+    JustText {
+        text: String,
+    },
+    #[serde(rename_all = "camelCase")]
+    WithMessage {
+        text: String,
+        message_id: u32,
+        author_display_name: String,
+        author_display_color: String,
+        override_sender_name: Option<String>,
+        image: Option<String>,
+        is_forwarded: bool,
+        view_type: MessageViewtype,
+    },
+}
+
+impl MessageObject {
+    pub async fn from_message_id(context: &Context, message_id: u32) -> Result<Self> {
+        let msg_id = MsgId::new(message_id);
+        Self::from_msg_id(context, msg_id).await
+    }
+
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
+
+        let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?;
+        let sender = ContactObject::try_from_dc_contact(context, sender_contact).await?;
+        let file_bytes = message.get_filebytes(context).await?.unwrap_or_default();
+        let override_sender_name = message.get_override_sender_name();
+
+        let webxdc_info = if message.get_viewtype() == Viewtype::Webxdc {
+            Some(WebxdcMessageInfo::get_for_message(context, msg_id).await?)
+        } else {
+            None
+        };
+
+        let parent_id = message.parent(context).await?.map(|m| m.get_id().to_u32());
+
+        let download_state = message.download_state().into();
+
+        let quote = if let Some(quoted_text) = message.quoted_text() {
+            match message.quoted_message(context).await? {
+                Some(quote) => {
+                    let quote_author = Contact::load_from_db(context, quote.get_from_id()).await?;
+                    Some(MessageQuote::WithMessage {
+                        text: quoted_text,
+                        message_id: quote.get_id().to_u32(),
+                        author_display_name: quote_author.get_display_name().to_owned(),
+                        author_display_color: color_int_to_hex_string(quote_author.get_color()),
+                        override_sender_name: quote.get_override_sender_name(),
+                        image: if quote.get_viewtype() == Viewtype::Image
+                            || quote.get_viewtype() == Viewtype::Gif
+                            || quote.get_viewtype() == Viewtype::Sticker
+                        {
+                            match quote.get_file(context) {
+                                Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+                                None => None,
+                            }
+                        } else {
+                            None
+                        },
+                        is_forwarded: quote.is_forwarded(),
+                        view_type: quote.get_viewtype().into(),
+                    })
+                }
+                None => Some(MessageQuote::JustText { text: quoted_text }),
+            }
+        } else {
+            None
+        };
+
+        let reactions = get_msg_reactions(context, msg_id).await?;
+        let reactions = if reactions.is_empty() {
+            None
+        } else {
+            Some(reactions.into())
+        };
+
+        Ok(MessageObject {
+            id: msg_id.to_u32(),
+            chat_id: message.get_chat_id().to_u32(),
+            from_id: message.get_from_id().to_u32(),
+            quote,
+            parent_id,
+            text: message.get_text(),
+            has_location: message.has_location(),
+            has_html: message.has_html(),
+            view_type: message.get_viewtype().into(),
+            state: message
+                .get_state()
+                .to_u32()
+                .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+            error: message.error(),
+
+            timestamp: message.get_timestamp(),
+            sort_timestamp: message.get_sort_timestamp(),
+            received_timestamp: message.get_received_timestamp(),
+            has_deviating_timestamp: message.has_deviating_timestamp(),
+
+            subject: message.get_subject().to_owned(),
+            show_padlock: message.get_showpadlock(),
+            is_setupmessage: message.is_setupmessage(),
+            is_info: message.is_info(),
+            is_forwarded: message.is_forwarded(),
+            is_bot: message.is_bot(),
+            system_message_type: message.get_info_type().into(),
+
+            duration: message.get_duration(),
+            dimensions_height: message.get_height(),
+            dimensions_width: message.get_width(),
+
+            videochat_type: match message.get_videochat_type() {
+                Some(vct) => Some(
+                    vct.to_u32()
+                        .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+                ),
+                None => None,
+            },
+            videochat_url: message.get_videochat_url(),
+
+            override_sender_name,
+            sender,
+
+            setup_code_begin: message.get_setupcodebegin(context).await,
+
+            file: match message.get_file(context) {
+                Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+                None => None,
+            }, //BLOBS
+            file_mime: message.get_filemime(),
+            file_bytes,
+            file_name: message.get_filename(),
+            webxdc_info,
+
+            download_state,
+
+            reactions,
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, TypeDef)]
+#[serde(rename = "Viewtype")]
+pub enum MessageViewtype {
+    Unknown,
+
+    /// Text message.
+    Text,
+
+    /// Image message.
+    /// If the image is an animated GIF, the type `Viewtype.Gif` should be used.
+    Image,
+
+    /// Animated GIF message.
+    Gif,
+
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker,
+
+    /// Message containing an Audio file.
+    Audio,
+
+    /// A voice message that was directly recorded by the user.
+    /// For all other audio messages, the type `Viewtype.Audio` should be used.
+    Voice,
+
+    /// Video messages.
+    Video,
+
+    /// Message containing any file, eg. a PDF.
+    File,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation,
+
+    /// Message is an webxdc instance.
+    Webxdc,
+}
+
+impl From<Viewtype> for MessageViewtype {
+    fn from(viewtype: Viewtype) -> Self {
+        match viewtype {
+            Viewtype::Unknown => MessageViewtype::Unknown,
+            Viewtype::Text => MessageViewtype::Text,
+            Viewtype::Image => MessageViewtype::Image,
+            Viewtype::Gif => MessageViewtype::Gif,
+            Viewtype::Sticker => MessageViewtype::Sticker,
+            Viewtype::Audio => MessageViewtype::Audio,
+            Viewtype::Voice => MessageViewtype::Voice,
+            Viewtype::Video => MessageViewtype::Video,
+            Viewtype::File => MessageViewtype::File,
+            Viewtype::VideochatInvitation => MessageViewtype::VideochatInvitation,
+            Viewtype::Webxdc => MessageViewtype::Webxdc,
+        }
+    }
+}
+
+impl From<MessageViewtype> for Viewtype {
+    fn from(viewtype: MessageViewtype) -> Self {
+        match viewtype {
+            MessageViewtype::Unknown => Viewtype::Unknown,
+            MessageViewtype::Text => Viewtype::Text,
+            MessageViewtype::Image => Viewtype::Image,
+            MessageViewtype::Gif => Viewtype::Gif,
+            MessageViewtype::Sticker => Viewtype::Sticker,
+            MessageViewtype::Audio => Viewtype::Audio,
+            MessageViewtype::Voice => Viewtype::Voice,
+            MessageViewtype::Video => Viewtype::Video,
+            MessageViewtype::File => Viewtype::File,
+            MessageViewtype::VideochatInvitation => Viewtype::VideochatInvitation,
+            MessageViewtype::Webxdc => Viewtype::Webxdc,
+        }
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+pub enum DownloadState {
+    Done,
+    Available,
+    Failure,
+    InProgress,
+}
+
+impl From<download::DownloadState> for DownloadState {
+    fn from(state: download::DownloadState) -> Self {
+        match state {
+            download::DownloadState::Done => DownloadState::Done,
+            download::DownloadState::Available => DownloadState::Available,
+            download::DownloadState::Failure => DownloadState::Failure,
+            download::DownloadState::InProgress => DownloadState::InProgress,
+        }
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+pub enum SystemMessageType {
+    Unknown,
+    GroupNameChanged,
+    GroupImageChanged,
+    MemberAddedToGroup,
+    MemberRemovedFromGroup,
+    AutocryptSetupMessage,
+    SecurejoinMessage,
+    LocationStreamingEnabled,
+    LocationOnly,
+
+    /// Chat ephemeral message timer is changed.
+    EphemeralTimerChanged,
+
+    // Chat protection state changed
+    ChatProtectionEnabled,
+    ChatProtectionDisabled,
+
+    /// Self-sent-message that contains only json used for multi-device-sync;
+    /// if possible, we attach that to other messages as for locations.
+    MultiDeviceSync,
+
+    // Sync message that contains a json payload
+    // sent to the other webxdc instances
+    // These messages are not shown in the chat.
+    WebxdcStatusUpdate,
+
+    /// Webxdc info added with `info` set in `send_webxdc_status_update()`.
+    WebxdcInfoMessage,
+}
+
+impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
+    fn from(system_message_type: deltachat::mimeparser::SystemMessage) -> Self {
+        use deltachat::mimeparser::SystemMessage;
+        match system_message_type {
+            SystemMessage::Unknown => SystemMessageType::Unknown,
+            SystemMessage::GroupNameChanged => SystemMessageType::GroupNameChanged,
+            SystemMessage::GroupImageChanged => SystemMessageType::GroupImageChanged,
+            SystemMessage::MemberAddedToGroup => SystemMessageType::MemberAddedToGroup,
+            SystemMessage::MemberRemovedFromGroup => SystemMessageType::MemberRemovedFromGroup,
+            SystemMessage::AutocryptSetupMessage => SystemMessageType::AutocryptSetupMessage,
+            SystemMessage::SecurejoinMessage => SystemMessageType::SecurejoinMessage,
+            SystemMessage::LocationStreamingEnabled => SystemMessageType::LocationStreamingEnabled,
+            SystemMessage::LocationOnly => SystemMessageType::LocationOnly,
+            SystemMessage::EphemeralTimerChanged => SystemMessageType::EphemeralTimerChanged,
+            SystemMessage::ChatProtectionEnabled => SystemMessageType::ChatProtectionEnabled,
+            SystemMessage::ChatProtectionDisabled => SystemMessageType::ChatProtectionDisabled,
+            SystemMessage::MultiDeviceSync => SystemMessageType::MultiDeviceSync,
+            SystemMessage::WebxdcStatusUpdate => SystemMessageType::WebxdcStatusUpdate,
+            SystemMessage::WebxdcInfoMessage => SystemMessageType::WebxdcInfoMessage,
+        }
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageNotificationInfo {
+    id: u32,
+    chat_id: u32,
+    account_id: u32,
+
+    image: Option<String>,
+    image_mime_type: Option<String>,
+
+    chat_name: String,
+    chat_profile_image: Option<String>,
+
+    /// also known as summary_text1
+    summary_prefix: Option<String>,
+    /// also known as summary_text2
+    summary_text: String,
+}
+
+impl MessageNotificationInfo {
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
+        let chat = Chat::load_from_db(context, message.get_chat_id()).await?;
+
+        let image = if matches!(
+            message.get_viewtype(),
+            Viewtype::Image | Viewtype::Gif | Viewtype::Sticker
+        ) {
+            message
+                .get_file(context)
+                .map(|path_buf| path_buf.to_str().map(|s| s.to_owned()))
+                .unwrap_or_default()
+        } else {
+            None
+        };
+
+        let chat_profile_image = chat
+            .get_profile_image(context)
+            .await?
+            .map(|path_buf| path_buf.to_str().map(|s| s.to_owned()))
+            .unwrap_or_default();
+
+        let summary = message.get_summary(context, Some(&chat)).await?;
+
+        Ok(MessageNotificationInfo {
+            id: msg_id.to_u32(),
+            chat_id: message.get_chat_id().to_u32(),
+            account_id: context.get_id(),
+            image,
+            image_mime_type: message.get_filemime(),
+            chat_name: chat.name,
+            chat_profile_image,
+            summary_prefix: summary.prefix.map(|s| s.to_string()),
+            summary_text: summary.text,
+        })
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageSearchResult {
+    id: u32,
+    author_profile_image: Option<String>,
+    author_name: String,
+    author_color: String,
+    chat_name: Option<String>,
+    message: String,
+    timestamp: i64,
+}
+
+impl MessageSearchResult {
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
+        let chat = Chat::load_from_db(context, message.get_chat_id()).await?;
+        let sender = Contact::load_from_db(context, message.get_from_id()).await?;
+
+        let profile_image = match sender.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+
+        Ok(Self {
+            id: msg_id.to_u32(),
+            author_profile_image: profile_image,
+            author_name: sender.get_display_name().to_owned(),
+            author_color: color_int_to_hex_string(sender.get_color()),
+            chat_name: if chat.get_type() == Chattype::Single {
+                Some(chat.get_name().to_owned())
+            } else {
+                None
+            },
+            message: message.get_text().unwrap_or_default(),
+            timestamp: message.get_timestamp(),
+        })
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase", rename = "MessageListItem", tag = "kind")]
+pub enum JSONRPCMessageListItem {
+    Message {
+        msg_id: u32,
+    },
+
+    /// Day marker, separating messages that correspond to different
+    /// days according to local time.
+    DayMarker {
+        /// Marker timestamp, for day markers, in unix milliseconds
+        timestamp: i64,
+    },
+}
+
+impl From<ChatItem> for JSONRPCMessageListItem {
+    fn from(item: ChatItem) -> Self {
+        match item {
+            ChatItem::Message { msg_id } => JSONRPCMessageListItem::Message {
+                msg_id: msg_id.to_u32(),
+            },
+            ChatItem::DayMarker { timestamp } => JSONRPCMessageListItem::DayMarker { timestamp },
+        }
+    }
+}
+
+#[derive(Deserialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageData {
+    pub text: Option<String>,
+    pub html: Option<String>,
+    pub viewtype: Option<MessageViewtype>,
+    pub file: Option<String>,
+    pub location: Option<(f64, f64)>,
+    pub override_sender_name: Option<String>,
+    pub quoted_message_id: Option<u32>,
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -0,0 +1,22 @@
+pub mod account;
+pub mod chat;
+pub mod chat_list;
+pub mod contact;
+pub mod location;
+pub mod message;
+pub mod provider_info;
+pub mod qr;
+pub mod reactions;
+pub mod webxdc;
+
+pub fn color_int_to_hex_string(color: u32) -> String {
+    format!("{color:#08x}").replace("0x", "#")
+}
+
+fn maybe_empty_string_to_option(string: String) -> Option<String> {
+    if string.is_empty() {
+        None
+    } else {
+        Some(string)
+    }
+}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -0,0 +1,22 @@
+use deltachat::provider::Provider;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct ProviderInfo {
+    pub before_login_hint: String,
+    pub overview_page: String,
+    pub status: u32, // in reality this is an enum, but for simplicity and because it gets converted into a number anyway, we use an u32 here.
+}
+
+impl ProviderInfo {
+    pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
+        provider.map(|p| ProviderInfo {
+            before_login_hint: p.before_login_hint.to_owned(),
+            overview_page: p.overview_page.to_owned(),
+            status: p.status.to_u32().unwrap(),
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/qr.rs

@@ -0,0 +1,213 @@
+use deltachat::qr::Qr;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Qr", rename_all = "camelCase")]
+#[serde(tag = "type")]
+pub enum QrObject {
+    AskVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    AskVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    FprOk {
+        contact_id: u32,
+    },
+    FprMismatch {
+        contact_id: Option<u32>,
+    },
+    FprWithoutAddr {
+        fingerprint: String,
+    },
+    Account {
+        domain: String,
+    },
+    WebrtcInstance {
+        domain: String,
+        instance_pattern: String,
+    },
+    Addr {
+        contact_id: u32,
+        draft: Option<String>,
+    },
+    Url {
+        url: String,
+    },
+    Text {
+        text: String,
+    },
+    WithdrawVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    WithdrawVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    ReviveVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    ReviveVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    Login {
+        address: String,
+    },
+}
+
+impl From<Qr> for QrObject {
+    fn from(qr: Qr) -> Self {
+        match qr {
+            Qr::AskVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::AskVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::AskVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::AskVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::FprOk { contact_id } => {
+                let contact_id = contact_id.to_u32();
+                QrObject::FprOk { contact_id }
+            }
+            Qr::FprMismatch { contact_id } => {
+                let contact_id = contact_id.map(|contact_id| contact_id.to_u32());
+                QrObject::FprMismatch { contact_id }
+            }
+            Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint },
+            Qr::Account { domain } => QrObject::Account { domain },
+            Qr::WebrtcInstance {
+                domain,
+                instance_pattern,
+            } => QrObject::WebrtcInstance {
+                domain,
+                instance_pattern,
+            },
+            Qr::Addr { contact_id, draft } => {
+                let contact_id = contact_id.to_u32();
+                QrObject::Addr { contact_id, draft }
+            }
+            Qr::Url { url } => QrObject::Url { url },
+            Qr::Text { text } => QrObject::Text { text },
+            Qr::WithdrawVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::WithdrawVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::WithdrawVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::WithdrawVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::ReviveVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::ReviveVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::ReviveVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::ReviveVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::Login { address, .. } => QrObject::Login { address },
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/reactions.rs

@@ -0,0 +1,47 @@
+use std::collections::BTreeMap;
+
+use deltachat::reaction::Reactions;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+/// Structure representing all reactions to a particular message.
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Reactions", rename_all = "camelCase")]
+pub struct JSONRPCReactions {
+    /// Map from a contact to it's reaction to message.
+    reactions_by_contact: BTreeMap<u32, Vec<String>>,
+    /// Unique reactions and their count
+    reactions: BTreeMap<String, u32>,
+}
+
+impl From<Reactions> for JSONRPCReactions {
+    fn from(reactions: Reactions) -> Self {
+        let mut reactions_by_contact: BTreeMap<u32, Vec<String>> = BTreeMap::new();
+        let mut unique_reactions: BTreeMap<String, u32> = BTreeMap::new();
+
+        for contact_id in reactions.contacts() {
+            let reaction = reactions.get(contact_id);
+            if reaction.is_empty() {
+                continue;
+            }
+            let emojis: Vec<String> = reaction
+                .emojis()
+                .into_iter()
+                .map(|emoji| emoji.to_owned())
+                .collect();
+            reactions_by_contact.insert(contact_id.to_u32(), emojis.clone());
+            for emoji in emojis {
+                if let Some(x) = unique_reactions.get_mut(&emoji) {
+                    *x += 1;
+                } else {
+                    unique_reactions.insert(emoji, 1);
+                }
+            }
+        }
+
+        JSONRPCReactions {
+            reactions_by_contact,
+            reactions: unique_reactions,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/webxdc.rs

@@ -0,0 +1,64 @@
+use deltachat::{
+    context::Context,
+    message::{Message, MsgId},
+    webxdc::WebxdcInfo,
+};
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::maybe_empty_string_to_option;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "WebxdcMessageInfo", rename_all = "camelCase")]
+pub struct WebxdcMessageInfo {
+    /// The name of the app.
+    ///
+    /// Defaults to the filename if not set in the manifest.
+    name: String,
+    /// App icon file name.
+    /// Defaults to an standard icon if nothing is set in the manifest.
+    ///
+    /// To get the file, use dc_msg_get_webxdc_blob(). (not yet in jsonrpc, use rust api or cffi for it)
+    ///
+    /// App icons should should be square,
+    /// the implementations will add round corners etc. as needed.
+    icon: String,
+    /// if the Webxdc represents a document, then this is the name of the document
+    document: Option<String>,
+    /// short string describing the state of the app,
+    /// sth. as "2 votes", "Highscore: 123",
+    /// can be changed by the apps
+    summary: Option<String>,
+    /// URL where the source code of the Webxdc and other information can be found;
+    /// defaults to an empty string.
+    /// Implementations may offer an menu or a button to open this URL.
+    source_code_url: Option<String>,
+    /// True if full internet access should be granted to the app.
+    internet_access: bool,
+}
+
+impl WebxdcMessageInfo {
+    pub async fn get_for_message(
+        context: &Context,
+        instance_message_id: MsgId,
+    ) -> anyhow::Result<Self> {
+        let message = Message::load_from_db(context, instance_message_id).await?;
+        let WebxdcInfo {
+            name,
+            icon,
+            document,
+            summary,
+            source_code_url,
+            internet_access,
+        } = message.get_webxdc_info(context).await?;
+
+        Ok(Self {
+            name,
+            icon,
+            document: maybe_empty_string_to_option(document),
+            summary: maybe_empty_string_to_option(summary),
+            source_code_url: maybe_empty_string_to_option(source_code_url),
+            internet_access,
+        })
+    }
+}

deltachat-jsonrpc/src/lib.rs

@@ -0,0 +1,93 @@
+pub mod api;
+pub use api::events;
+pub use yerpc;
+
+#[cfg(test)]
+mod tests {
+    use async_channel::unbounded;
+    use futures::StreamExt;
+    use tempfile::TempDir;
+    use yerpc::{RpcClient, RpcSession};
+
+    use super::api::{Accounts, CommandApi};
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
+        let tmp_dir = TempDir::new().unwrap().path().into();
+        let accounts = Accounts::new(tmp_dir).await?;
+        let api = CommandApi::new(accounts);
+
+        let (sender, mut receiver) = unbounded::<String>();
+
+        let (client, mut rx) = RpcClient::new();
+        let session = RpcSession::new(client, api);
+        tokio::spawn({
+            async move {
+                while let Some(message) = rx.next().await {
+                    let message = serde_json::to_string(&message)?;
+                    sender.send(message).await?;
+                }
+                let res: Result<(), anyhow::Error> = Ok(());
+                res
+            }
+        });
+
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
+            let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            println!("{result:?}");
+            assert_eq!(result, Some(response.to_owned()));
+        }
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
+            let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            println!("{result:?}");
+            assert_eq!(result, Some(response.to_owned()));
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_batch_set_config() -> anyhow::Result<()> {
+        let tmp_dir = TempDir::new().unwrap().path().into();
+        let accounts = Accounts::new(tmp_dir).await?;
+        let api = CommandApi::new(accounts);
+
+        let (sender, mut receiver) = unbounded::<String>();
+
+        let (client, mut rx) = RpcClient::new();
+        let session = RpcSession::new(client, api);
+        tokio::spawn({
+            async move {
+                while let Some(message) = rx.next().await {
+                    let message = serde_json::to_string(&message)?;
+                    sender.send(message).await?;
+                }
+                let res: Result<(), anyhow::Error> = Ok(());
+                res
+            }
+        });
+
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
+            let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
+        }
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
+            let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
+        }
+
+        Ok(())
+    }
+}

deltachat-jsonrpc/src/webserver.rs

@@ -0,0 +1,55 @@
+use std::net::SocketAddr;
+use std::path::PathBuf;
+
+use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router};
+use yerpc::axum::handle_ws_rpc;
+use yerpc::{RpcClient, RpcSession};
+
+mod api;
+use api::events::event_to_json_rpc_notification;
+use api::{Accounts, CommandApi};
+
+const DEFAULT_PORT: u16 = 20808;
+
+#[tokio::main(flavor = "multi_thread")]
+async fn main() -> Result<(), std::io::Error> {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "./accounts".to_string());
+    let port = std::env::var("DC_PORT")
+        .map(|port| port.parse::<u16>().expect("DC_PORT must be a number"))
+        .unwrap_or(DEFAULT_PORT);
+    log::info!("Starting with accounts directory `{path}`.");
+    let accounts = Accounts::new(PathBuf::from(&path)).await.unwrap();
+    let state = CommandApi::new(accounts);
+
+    let app = Router::new()
+        .route("/ws", get(handler))
+        .layer(Extension(state.clone()));
+
+    tokio::spawn(async move {
+        state.accounts.read().await.start_io().await;
+    });
+
+    let addr = SocketAddr::from(([127, 0, 0, 1], port));
+    log::info!("JSON-RPC WebSocket server listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .await
+        .unwrap();
+
+    Ok(())
+}
+
+async fn handler(ws: WebSocketUpgrade, Extension(api): Extension<CommandApi>) -> Response {
+    let (client, out_receiver) = RpcClient::new();
+    let session = RpcSession::new(client.clone(), api.clone());
+    tokio::spawn(async move {
+        let events = api.accounts.read().await.get_event_emitter();
+        while let Some(event) = events.recv().await {
+            let event = event_to_json_rpc_notification(event);
+            client.send_notification("event", Some(event)).await.ok();
+        }
+    });
+    handle_ws_rpc(ws, out_receiver, session).await
+}

deltachat-jsonrpc/typescript/.gitignore

@@ -0,0 +1,9 @@
+node_modules
+dist
+test_dist
+coverage
+yarn.lock
+package-lock.json
+docs
+accounts
+generated

deltachat-jsonrpc/typescript/.npmignore

@@ -0,0 +1,12 @@
+node_modules
+accounts
+docs
+coverage
+yarn*
+package-lock.json
+.prettierignore
+example.html
+report_api_coverage.mjs
+scripts
+dist/example
+dist/test

deltachat-jsonrpc/typescript/.prettierignore

@@ -0,0 +1,3 @@
+coverage
+dist
+generated

deltachat-jsonrpc/typescript/deltachat.ts

@@ -0,0 +1 @@
+export * from "./src/lib.js";

deltachat-jsonrpc/typescript/example.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>DeltaChat JSON-RPC example</title>
+    <style>
+      body {
+        font-family: monospace;
+        background: black;
+        color: grey;
+      }
+      .grid {
+        display: grid;
+        grid-template-columns: 3fr 1fr;
+        grid-template-areas: "a a" "b c";
+      }
+      .message {
+        color: red;
+      }
+      #header {
+        grid-area: a;
+        color: white;
+        font-size: 1.2rem;
+      }
+      #header a {
+        color: white;
+        font-weight: bold;
+      }
+      #main {
+        grid-area: b;
+        color: green;
+      }
+      #main h2,
+      #main h3 {
+        color: blue;
+      }
+      #side {
+        grid-area: c;
+        color: #777;
+        overflow-y: auto;
+      }
+    </style>
+    <script type="module" src="dist/example.bundle.js"></script>
+  </head>
+  <body>
+    <h1>DeltaChat JSON-RPC example</h1>
+    <div class="grid">
+      <div id="header"></div>
+      <div id="main"></div>
+      <div id="side"><h2>log</h2></div>
+    </div>
+    <p>
+      Tip: open the dev console and use the client with
+      <code>window.client</code>
+    </p>
+  </body>
+</html>

deltachat-jsonrpc/typescript/example/example.ts

@@ -0,0 +1,110 @@
+import { DcEvent, DeltaChat } from "../deltachat.js";
+
+var SELECTED_ACCOUNT = 0;
+
+window.addEventListener("DOMContentLoaded", (_event) => {
+  (window as any).selectDeltaAccount = (id: string) => {
+    SELECTED_ACCOUNT = Number(id);
+    window.dispatchEvent(new Event("account-changed"));
+  };
+  console.log("launch run script...");
+  run().catch((err) => console.error("run failed", err));
+});
+
+async function run() {
+  const $main = document.getElementById("main")!;
+  const $side = document.getElementById("side")!;
+  const $head = document.getElementById("header")!;
+
+  const client = new DeltaChat("ws://localhost:20808/ws");
+
+  (window as any).client = client.rpc;
+
+  client.on("ALL", (accountId, event) => {
+    onIncomingEvent(accountId, event);
+  });
+
+  window.addEventListener("account-changed", async (_event: Event) => {
+    listChatsForSelectedAccount();
+  });
+
+  await Promise.all([loadAccountsInHeader(), listChatsForSelectedAccount()]);
+
+  async function loadAccountsInHeader() {
+    console.log("load accounts");
+    const accounts = await client.rpc.getAllAccounts();
+    console.log("accounts loaded", accounts);
+    for (const account of accounts) {
+      if (account.type === "Configured") {
+        write(
+          $head,
+          `<a href="#" onclick="selectDeltaAccount(${account.id})">
+          ${account.id}: ${account.addr!}
+          </a>&nbsp;`
+        );
+      } else {
+        write(
+          $head,
+          `<a href="#">
+          ${account.id}: (unconfigured)
+          </a>&nbsp;`
+        );
+      }
+    }
+  }
+
+  async function listChatsForSelectedAccount() {
+    clear($main);
+    const selectedAccount = SELECTED_ACCOUNT;
+    const info = await client.rpc.getAccountInfo(selectedAccount);
+    if (info.type !== "Configured") {
+      return write($main, "Account is not configured");
+    }
+    write($main, `<h2>${info.addr!}</h2>`);
+    const chats = await client.rpc.getChatlistEntries(
+      selectedAccount,
+      0,
+      null,
+      null
+    );
+    for (const [chatId, _messageId] of chats) {
+      const chat = await client.rpc.getFullChatById(selectedAccount, chatId);
+      write($main, `<h3>${chat.name}</h3>`);
+      const messageIds = await client.rpc.getMessageIds(
+        selectedAccount,
+        chatId,
+        false,
+        false
+      );
+      const messages = await client.rpc.getMessages(
+        selectedAccount,
+        messageIds
+      );
+      for (const [_messageId, message] of Object.entries(messages)) {
+        if (message.variant === "message")
+          write($main, `<p>${message.text}</p>`);
+        else write($main, `<p>loading error: ${message.error}</p>`);
+      }
+    }
+  }
+
+  function onIncomingEvent(accountId: number, event: DcEvent) {
+    write(
+      $side,
+      `
+        <p class="message">
+          [<strong>${event.type}</strong> on account ${accountId}]<br>
+          <em>f1:</em> ${JSON.stringify(
+            Object.assign({}, event, { type: undefined })
+          )}
+        </p>`
+    );
+  }
+}
+
+function write(el: HTMLElement, html: string) {
+  el.innerHTML += html;
+}
+function clear(el: HTMLElement) {
+  el.innerHTML = "";
+}

deltachat-jsonrpc/typescript/example/node-add-account.js

@@ -0,0 +1,29 @@
+import { DeltaChat } from "../dist/deltachat.js";
+
+run().catch(console.error);
+
+async function run() {
+  const delta = new DeltaChat("ws://localhost:20808/ws");
+  delta.on("event", (event) => {
+    console.log("event", event.data);
+  });
+
+  const email = process.argv[2];
+  const password = process.argv[3];
+  if (!email || !password)
+    throw new Error(
+      "USAGE: node node-add-account.js <EMAILADDRESS> <PASSWORD>"
+    );
+  console.log(`creating account for ${email}`);
+  const id = await delta.rpc.addAccount();
+  console.log(`created account id ${id}`);
+  await delta.rpc.setConfig(id, "addr", email);
+  await delta.rpc.setConfig(id, "mail_pw", password);
+  console.log("configuration updated");
+  await delta.rpc.configure(id);
+  console.log("account configured!");
+
+  const accounts = await delta.rpc.getAllAccounts();
+  console.log("accounts", accounts);
+  console.log("waiting for events...");
+}

deltachat-jsonrpc/typescript/example/node-demo.js

@@ -0,0 +1,14 @@
+import { DeltaChat } from "../dist/deltachat.js";
+
+run().catch(console.error);
+
+async function run() {
+  const delta = new DeltaChat();
+  delta.on("event", (event) => {
+    console.log("event", event.data);
+  });
+
+  const accounts = await delta.rpc.getAllAccounts();
+  console.log("accounts", accounts);
+  console.log("waiting for events...");
+}

deltachat-jsonrpc/typescript/package.json

@@ -0,0 +1,59 @@
+{
+  "author": "Delta Chat Developers (ML) <delta@codespeak.net>",
+  "dependencies": {
+    "@deltachat/tiny-emitter": "3.0.0",
+    "isomorphic-ws": "^4.0.1",
+    "yerpc": "^0.4.3"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.2.21",
+    "@types/chai-as-promised": "^7.1.5",
+    "@types/mocha": "^9.0.0",
+    "@types/node-fetch": "^2.5.7",
+    "@types/ws": "^7.2.4",
+    "c8": "^7.10.0",
+    "chai": "^4.3.4",
+    "chai-as-promised": "^7.1.1",
+    "esbuild": "^0.17.9",
+    "http-server": "^14.1.1",
+    "mocha": "^9.1.1",
+    "node-fetch": "^2.6.1",
+    "npm-run-all": "^4.1.5",
+    "prettier": "^2.6.2",
+    "typedoc": "^0.23.2",
+    "typescript": "^4.5.5",
+    "ws": "^8.5.0"
+  },
+  "exports": {
+    ".": {
+      "import": "./dist/deltachat.js",
+      "require": "./dist/deltachat.cjs"
+    }
+  },
+  "license": "MPL-2.0",
+  "main": "dist/deltachat.js",
+  "name": "@deltachat/jsonrpc-client",
+  "scripts": {
+    "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
+    "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
+    "build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs",
+    "build:tsc": "tsc",
+    "docs": "typedoc --out docs deltachat.ts",
+    "example": "run-s build example:build example:start",
+    "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js",
+    "example:dev": "esbuild example/example.ts --bundle --outfile=dist/example.bundle.js --servedir=.",
+    "example:start": "http-server .",
+    "extract-constants": "node ./scripts/generate-constants.js",
+    "generate-bindings": "cargo test",
+    "prettier:check": "prettier --check .",
+    "prettier:fix": "prettier --write .",
+    "test": "run-s test:prepare test:run-coverage test:report-coverage",
+    "test:prepare": "cargo build --package deltachat-rpc-server --bin deltachat-rpc-server",
+    "test:report-coverage": "node report_api_coverage.mjs",
+    "test:run": "mocha dist/test",
+    "test:run-coverage": "COVERAGE=1 NODE_OPTIONS=--enable-source-maps c8 --include 'dist/*' -r text -r html -r json mocha dist/test"
+  },
+  "type": "module",
+  "types": "dist/deltachat.d.ts",
+  "version": "1.111.0"
+}

deltachat-jsonrpc/typescript/report_api_coverage.mjs

@@ -0,0 +1,28 @@
+import { readFileSync } from "fs";
+// only checks for the coverage of the api functions in bindings.ts for now
+const generatedFile = "typescript/generated/client.ts";
+const json = JSON.parse(readFileSync("./coverage/coverage-final.json"));
+const jsonCoverage =
+  json[Object.keys(json).find((k) => k.includes(generatedFile))];
+const fnMap = Object.keys(jsonCoverage.fnMap).map(
+  (key) => jsonCoverage.fnMap[key]
+);
+const htmlCoverage = readFileSync(
+  "./coverage/" + generatedFile + ".html",
+  "utf8"
+);
+const uncoveredLines = htmlCoverage
+  .split("\n")
+  .filter((line) => line.includes(`"function not covered"`));
+const uncoveredFunctions = uncoveredLines.map(
+  (line) => />([\w_]+)\(/.exec(line)[1]
+);
+console.log(
+  "\nUncovered api functions:\n" +
+    uncoveredFunctions
+      .map((uF) => fnMap.find(({ name }) => name === uF))
+      .map(
+        ({ name, line }) => `.${name.padEnd(40)}  (${generatedFile}:${line})`
+      )
+      .join("\n")
+);

deltachat-jsonrpc/typescript/scripts/generate-constants.js

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+import { readFileSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { fileURLToPath } from "url";
+import { dirname } from "path";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const data = [];
+const header = resolve(__dirname, "../../../deltachat-ffi/deltachat.h");
+
+console.log("Generating constants...");
+
+const header_data = readFileSync(header, "UTF-8");
+const regex = /^#define\s+(\w+)\s+(\w+)/gm;
+let match;
+while (null != (match = regex.exec(header_data))) {
+  const key = match[1];
+  const value = parseInt(match[2]);
+  if (!isNaN(value)) {
+    data.push({ key, value });
+  }
+}
+
+const constants = data
+  .filter(
+    ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
+  )
+  .sort((lhs, rhs) => {
+    if (lhs.key < rhs.key) return -1;
+    else if (lhs.key > rhs.key) return 1;
+    return 0;
+  })
+  .filter(({ key }) => {
+    // filter out what we don't need it
+    return !(
+      key.startsWith("DC_EVENT_") ||
+      key.startsWith("DC_IMEX_") ||
+      key.startsWith("DC_CHAT_VISIBILITY") ||
+      key.startsWith("DC_DOWNLOAD") ||
+      key.startsWith("DC_INFO_") ||
+      (key.startsWith("DC_MSG") && !key.startsWith("DC_MSG_ID")) ||
+      key.startsWith("DC_QR_")
+    );
+  })
+  .map((row) => {
+    return `  ${row.key}: ${row.value}`;
+  })
+  .join(",\n");
+
+writeFileSync(
+  resolve(__dirname, "../generated/constants.ts"),
+  `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, " =")},\n}\n`
+);

deltachat-jsonrpc/typescript/src/client.ts

@@ -0,0 +1,127 @@
+import * as T from "../generated/types.js";
+import * as RPC from "../generated/jsonrpc.js";
+import { RawClient } from "../generated/client.js";
+import { Event } from "../generated/events.js";
+import { WebsocketTransport, BaseTransport, Request } from "yerpc";
+import { TinyEmitter } from "@deltachat/tiny-emitter";
+
+type DCWireEvent<T extends Event> = {
+  event: T;
+  contextId: number;
+};
+// export type Events = Record<
+//   Event["type"] | "ALL",
+//   (event: DeltaChatEvent<Event>) => void
+// >;
+
+type Events = { ALL: (accountId: number, event: Event) => void } & {
+  [Property in Event["type"]]: (
+    accountId: number,
+    event: Extract<Event, { type: Property }>
+  ) => void;
+};
+
+type ContextEvents = { ALL: (event: Event) => void } & {
+  [Property in Event["type"]]: (
+    event: Extract<Event, { type: Property }>
+  ) => void;
+};
+
+export type DcEvent = Event;
+export type DcEventType<T extends Event["type"]> = Extract<Event, { type: T }>;
+
+export class BaseDeltaChat<
+  Transport extends BaseTransport<any>
+> extends TinyEmitter<Events> {
+  rpc: RawClient;
+  account?: T.Account;
+  private contextEmitters: { [key: number]: TinyEmitter<ContextEvents> } = {};
+  constructor(public transport: Transport) {
+    super();
+    this.rpc = new RawClient(this.transport);
+    this.transport.on("request", (request: Request) => {
+      const method = request.method;
+      if (method === "event") {
+        const event = request.params! as DCWireEvent<Event>;
+        //@ts-ignore
+        this.emit(event.event.type, event.contextId, event.event as any);
+        this.emit("ALL", event.contextId, event.event as any);
+
+        if (this.contextEmitters[event.contextId]) {
+          this.contextEmitters[event.contextId].emit(
+            event.event.type,
+            //@ts-ignore
+            event.event as any
+          );
+          this.contextEmitters[event.contextId].emit("ALL", event.event);
+        }
+      }
+    });
+  }
+
+  async listAccounts(): Promise<T.Account[]> {
+    return await this.rpc.getAllAccounts();
+  }
+
+  getContextEvents(account_id: number) {
+    if (this.contextEmitters[account_id]) {
+      return this.contextEmitters[account_id];
+    } else {
+      this.contextEmitters[account_id] = new TinyEmitter();
+      return this.contextEmitters[account_id];
+    }
+  }
+}
+
+export type Opts = {
+  url: string;
+};
+
+export const DEFAULT_OPTS: Opts = {
+  url: "ws://localhost:20808/ws",
+};
+export class DeltaChat extends BaseDeltaChat<WebsocketTransport> {
+  opts: Opts;
+  close() {
+    this.transport.close();
+  }
+  constructor(opts?: Opts | string) {
+    if (typeof opts === "string") opts = { url: opts };
+    if (opts) opts = { ...DEFAULT_OPTS, ...opts };
+    else opts = { ...DEFAULT_OPTS };
+    const transport = new WebsocketTransport(opts.url);
+    super(transport);
+    this.opts = opts;
+  }
+}
+
+export class StdioDeltaChat extends BaseDeltaChat<StdioTransport> {
+  close() {}
+  constructor(input: any, output: any) {
+    const transport = new StdioTransport(input, output);
+    super(transport);
+  }
+}
+
+export class StdioTransport extends BaseTransport {
+  constructor(public input: any, public output: any) {
+    super();
+
+    var buffer = "";
+    this.output.on("data", (data: any) => {
+      buffer += data.toString();
+      while (buffer.includes("\n")) {
+        const n = buffer.indexOf("\n");
+        const line = buffer.substring(0, n);
+        const message = JSON.parse(line);
+        this._onmessage(message);
+        buffer = buffer.substring(n + 1);
+      }
+    });
+  }
+
+  _send(message: any): void {
+    const serialized = JSON.stringify(message);
+    this.input.write(serialized + "\n");
+  }
+}

deltachat-jsonrpc/typescript/src/lib.ts

@@ -0,0 +1,7 @@
+export * as RPC from "../generated/jsonrpc.js";
+export * as T from "../generated/types.js";
+export * from "../generated/events.js";
+export { RawClient } from "../generated/client.js";
+export * from "./client.js";
+export * as yerpc from "yerpc";
+export { C } from "../generated/constants.js";

deltachat-jsonrpc/typescript/test/basic.ts

@@ -0,0 +1,158 @@
+import { strictEqual } from "assert";
+import chai, { assert, expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+chai.use(chaiAsPromised);
+import { StdioDeltaChat as DeltaChat } from "../deltachat.js";
+
+import { RpcServerHandle, startServer } from "./test_base.js";
+
+describe("basic tests", () => {
+  let serverHandle: RpcServerHandle;
+  let dc: DeltaChat;
+
+  before(async () => {
+    serverHandle = await startServer();
+    dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout);
+    // dc.on("ALL", (event) => {
+    //console.log("event", event);
+    // });
+  });
+
+  after(async () => {
+    dc && dc.close();
+    await serverHandle.close();
+  });
+
+  it("check email address validity", async () => {
+    const validAddresses = [
+      "email@example.com",
+      "36aa165ae3406424e0c61af17700f397cad3fe8ab83d682d0bddf3338a5dd52e@yggmail@yggmail",
+    ];
+    const invalidAddresses = ["email@", "example.com", "emai221"];
+
+    expect(
+      await Promise.all(
+        validAddresses.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(false);
+
+    expect(
+      await Promise.all(
+        invalidAddresses.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(true);
+  });
+
+  it("system info", async () => {
+    const systemInfo = await dc.rpc.getSystemInfo();
+    expect(systemInfo).to.contain.keys([
+      "arch",
+      "num_cpus",
+      "deltachat_core_version",
+      "sqlite_version",
+    ]);
+  });
+
+  describe("account management", () => {
+    it("should create account", async () => {
+      const res = await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 1);
+    });
+
+    it("should remove the account again", async () => {
+      await dc.rpc.removeAccount((await dc.rpc.getAllAccountIds())[0]);
+      assert((await dc.rpc.getAllAccountIds()).length === 0);
+    });
+
+    it("should create multiple accounts", async () => {
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 4);
+    });
+  });
+
+  describe("contact management", function () {
+    let accountId: number;
+    before(async () => {
+      accountId = await dc.rpc.addAccount();
+    });
+    it("should block and unblock contact", async function () {
+      const contactId = await dc.rpc.createContact(
+        accountId,
+        "example@delta.chat",
+        null
+      );
+      expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
+        .false;
+      await dc.rpc.blockContact(accountId, contactId);
+      expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
+        .true;
+      expect(await dc.rpc.getBlockedContacts(accountId)).to.have.length(1);
+      await dc.rpc.unblockContact(accountId, contactId);
+      expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
+        .false;
+      expect(await dc.rpc.getBlockedContacts(accountId)).to.have.length(0);
+    });
+  });
+
+  describe("configuration", function () {
+    let accountId: number;
+    before(async () => {
+      accountId = await dc.rpc.addAccount();
+    });
+
+    it("set and retrieve", async function () {
+      await dc.rpc.setConfig(accountId, "addr", "valid@email");
+      assert((await dc.rpc.getConfig(accountId, "addr")) == "valid@email");
+    });
+    it("set invalid key should throw", async function () {
+      await expect(dc.rpc.setConfig(accountId, "invalid_key", "some value")).to
+        .be.eventually.rejected;
+    });
+    it("get invalid key should throw", async function () {
+      await expect(dc.rpc.getConfig(accountId, "invalid_key")).to.be.eventually
+        .rejected;
+    });
+    it("set and retrieve ui.*", async function () {
+      await dc.rpc.setConfig(accountId, "ui.chat_bg", "color:red");
+      assert((await dc.rpc.getConfig(accountId, "ui.chat_bg")) == "color:red");
+    });
+    it("set and retrieve (batch)", async function () {
+      const config = { addr: "valid@email", mail_pw: "1234" };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrieve ui.* (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:green",
+        "ui.enter_key_sends": "true",
+      };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrieve mixed(ui and core) (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:yellow",
+        "ui.enter_key_sends": "false",
+        addr: "valid2@email",
+        mail_pw: "123456",
+      };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(
+        accountId,
+        Object.keys(config)
+      );
+      expect(retrieved).to.deep.equal(config);
+    });
+  });
+});

deltachat-jsonrpc/typescript/test/online.ts

@@ -0,0 +1,200 @@
+import { assert, expect } from "chai";
+import { StdioDeltaChat as DeltaChat, DcEvent } from "../deltachat.js";
+import { RpcServerHandle, createTempUser, startServer } from "./test_base.js";
+
+const EVENT_TIMEOUT = 20000;
+
+describe("online tests", function () {
+  let serverHandle: RpcServerHandle;
+  let dc: DeltaChat;
+  let account1: { email: string; password: string };
+  let account2: { email: string; password: string };
+  let accountId1: number, accountId2: number;
+
+  before(async function () {
+    this.timeout(60000);
+    if (!process.env.DCC_NEW_TMP_EMAIL) {
+      if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
+        console.error(
+          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
+          "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
+        );
+        process.exit(1);
+      }
+      console.log(
+        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests"
+      );
+      this.skip();
+    }
+    serverHandle = await startServer();
+    dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout);
+
+    dc.on("ALL", (contextId, { type }) => {
+      if (type !== "Info") console.log(contextId, type);
+    });
+
+    account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account1 || !account1.email || !account1.password) {
+      console.log(
+        "We didn't got back an account from the api, skip integration tests"
+      );
+      this.skip();
+    }
+
+    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account2 || !account2.email || !account2.password) {
+      console.log(
+        "We didn't got back an account2 from the api, skip integration tests"
+      );
+      this.skip();
+    }
+  });
+
+  after(async () => {
+    dc && dc.close();
+    serverHandle && (await serverHandle.close());
+  });
+
+  let accountsConfigured = false;
+
+  it("configure test accounts", async function () {
+    this.timeout(40000);
+
+    accountId1 = await dc.rpc.addAccount();
+    await dc.rpc.setConfig(accountId1, "addr", account1.email);
+    await dc.rpc.setConfig(accountId1, "mail_pw", account1.password);
+    await dc.rpc.configure(accountId1);
+
+    accountId2 = await dc.rpc.addAccount();
+    await dc.rpc.batchSetConfig(accountId2, {
+      addr: account2.email,
+      mail_pw: account2.password,
+    });
+    await dc.rpc.configure(accountId2);
+    accountsConfigured = true;
+  });
+
+  it("send and receive text message", async function () {
+    if (!accountsConfigured) {
+      this.skip();
+    }
+    this.timeout(15000);
+
+    const contactId = await dc.rpc.createContact(
+      accountId1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
+
+    await dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello");
+    const { chatId: chatIdOnAccountB } = await eventPromise;
+    await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
+    const messageList = await dc.rpc.getMessageIds(
+      accountId2,
+      chatIdOnAccountB,
+      false,
+      false
+    );
+
+    expect(messageList).have.length(1);
+    const message = await dc.rpc.getMessage(accountId2, messageList[0]);
+    expect(message.text).equal("Hello");
+  });
+
+  it("send and receive text message roundtrip, encrypted on answer onwards", async function () {
+    if (!accountsConfigured) {
+      this.skip();
+    }
+    this.timeout(10000);
+
+    // send message from A to B
+    const contactId = await dc.rpc.createContact(
+      accountId1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
+    dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello2");
+    // wait for message from A
+    console.log("wait for message from A");
+
+    const event = await eventPromise;
+    const { chatId: chatIdOnAccountB } = event;
+
+    await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
+    const messageList = await dc.rpc.getMessageIds(
+      accountId2,
+      chatIdOnAccountB,
+      false,
+      false
+    );
+    const message = await dc.rpc.getMessage(
+      accountId2,
+      messageList.reverse()[0]
+    );
+    expect(message.text).equal("Hello2");
+    // Send message back from B to A
+    const eventPromise2 = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId1),
+      waitForEvent(dc, "IncomingMsg", accountId1),
+    ]);
+    dc.rpc.miscSendTextMessage(accountId2, chatId, "super secret message");
+    // Check if answer arives at A and if it is encrypted
+    await eventPromise2;
+
+    const messageId = (
+      await dc.rpc.getMessageIds(accountId1, chatId, false, false)
+    ).reverse()[0];
+    const message2 = await dc.rpc.getMessage(accountId1, messageId);
+    expect(message2.text).equal("super secret message");
+    expect(message2.showPadlock).equal(true);
+  });
+
+  it("get provider info for example.com", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info = await dc.rpc.getProviderInfo(acc, "example.com");
+    expect(info).to.be.not.null;
+    expect(info?.overviewPage).to.equal(
+      "https://providers.delta.chat/example-com"
+    );
+    expect(info?.status).to.equal(3);
+  });
+
+  it("get provider info - domain and email should give same result", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info_domain = await dc.rpc.getProviderInfo(acc, "example.com");
+    const info_email = await dc.rpc.getProviderInfo(acc, "hi@example.com");
+    expect(info_email).to.deep.equal(info_domain);
+  });
+});
+
+async function waitForEvent<T extends DcEvent["type"]>(
+  dc: DeltaChat,
+  eventType: T,
+  accountId: number,
+  timeout: number = EVENT_TIMEOUT
+): Promise<Extract<DcEvent, { type: T }>> {
+  return new Promise((resolve, reject) => {
+    const rejectTimeout = setTimeout(
+      () => reject(new Error("Timeout reached before event came in")),
+      timeout
+    );
+    const callback = (contextId: number, event: DcEvent) => {
+      if (contextId == accountId) {
+        dc.off(eventType, callback);
+        clearTimeout(rejectTimeout);
+        resolve(event as any);
+      }
+    };
+    dc.on(eventType, callback);
+  });
+}

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -0,0 +1,91 @@
+import { tmpdir } from "os";
+import { join, resolve } from "path";
+import { mkdtemp, rm } from "fs/promises";
+import { spawn, exec } from "child_process";
+import fetch from "node-fetch";
+import { Readable, Writable } from "node:stream";
+
+export type RpcServerHandle = {
+  stdin: Writable;
+  stdout: Readable;
+  close: () => Promise<void>;
+};
+
+export async function startServer(): Promise<RpcServerHandle> {
+  const tmpDir = await mkdtemp(join(tmpdir(), "deltachat-jsonrpc-test"));
+
+  const pathToServerBinary = resolve(
+    join(await getTargetDir(), "debug/deltachat-rpc-server")
+  );
+
+  const server = spawn(pathToServerBinary, {
+    cwd: tmpDir,
+    env: {
+      RUST_LOG: process.env.RUST_LOG || "info",
+      RUST_MIN_STACK: "8388608",
+    },
+  });
+
+  server.on("error", (err) => {
+    throw new Error(
+      "Failed to start server executable " +
+        pathToServerBinary +
+        ", make sure you built it first."
+    );
+  });
+  let shouldClose = false;
+
+  server.on("exit", () => {
+    if (shouldClose) {
+      return;
+    }
+    throw new Error("Server quit");
+  });
+
+  server.stderr.pipe(process.stderr);
+
+  return {
+    stdin: server.stdin,
+    stdout: server.stdout,
+    close: async () => {
+      shouldClose = true;
+      if (!server.kill()) {
+        console.log("server termination failed");
+      }
+      await rm(tmpDir, { recursive: true });
+    },
+  };
+}
+
+export async function createTempUser(url: string) {
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "cache-control": "no-cache",
+    },
+  });
+  if (!response.ok) throw new Error("Received invalid response");
+  return response.json();
+}
+
+function getTargetDir(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    exec(
+      "cargo metadata --no-deps --format-version 1",
+      (error, stdout, _stderr) => {
+        if (error) {
+          console.log("error", error);
+          reject(error);
+        } else {
+          try {
+            const json = JSON.parse(stdout);
+            resolve(json.target_directory);
+          } catch (error) {
+            console.log("json error", error);
+            reject(error);
+          }
+        }
+      }
+    );
+  });
+}

deltachat-jsonrpc/typescript/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "alwaysStrict": true,
+    "strict": true,
+    "sourceMap": true,
+    "strictNullChecks": true,
+    "rootDir": ".",
+    "outDir": "dist",
+    "lib": ["ES2017", "dom"],
+    "target": "ES2017",
+    "module": "es2020",
+    "declaration": true,
+    "esModuleInterop": true,
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "isolatedModules": true
+  },
+  "include": ["*.ts", "example/*.ts", "test/*.ts"],
+  "compileOnSave": false
+}

deltachat-ratelimit/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "ratelimit"
+version = "1.0.0"
+description = "Token bucket implementation"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]

deltachat-ratelimit/src/lib.rs

@@ -0,0 +1,134 @@
+//! # Rate limiting module.
+//!
+//! This module contains implementation of token bucket policy.
+//! Its primary use is preventing Delta Chat from sending too many messages, especially automatic,
+//! such as read receipts.
+
+use std::time::{Duration, SystemTime};
+
+#[derive(Debug)]
+pub struct Ratelimit {
+    /// Time of the last update.
+    last_update: SystemTime,
+
+    /// Number of messages sent within the time window ending at `last_update`.
+    current_value: f64,
+
+    /// Time window size.
+    window: Duration,
+
+    /// Number of messages allowed to send within the time window.
+    quota: f64,
+}
+
+impl Ratelimit {
+    /// Returns a new rate limiter with the given constraints.
+    ///
+    /// Rate limiter will allow to send no more than `quota` messages within duration `window`.
+    pub fn new(window: Duration, quota: f64) -> Self {
+        Self::new_at(window, quota, SystemTime::now())
+    }
+
+    /// Returns a new rate limiter with given current time for testing purposes.
+    fn new_at(window: Duration, quota: f64, now: SystemTime) -> Self {
+        Self {
+            last_update: now,
+            current_value: 0.0,
+            window,
+            quota,
+        }
+    }
+
+    /// Returns current number of sent messages.
+    fn current_value_at(&self, now: SystemTime) -> f64 {
+        let rate: f64 = self.quota / self.window.as_secs_f64();
+        let elapsed = now
+            .duration_since(self.last_update)
+            .unwrap_or(Duration::ZERO)
+            .as_secs_f64();
+        f64::max(0.0, self.current_value - rate * elapsed)
+    }
+
+    /// Returns true if it is allowed to send a message.
+    fn can_send_at(&self, now: SystemTime) -> bool {
+        self.current_value_at(now) + 1.0 <= self.quota
+    }
+
+    /// Returns true if can send another message now.
+    ///
+    /// This method takes mutable reference
+    pub fn can_send(&self) -> bool {
+        self.can_send_at(SystemTime::now())
+    }
+
+    fn send_at(&mut self, now: SystemTime) {
+        self.current_value = f64::min(self.quota, self.current_value_at(now) + 1.0);
+        self.last_update = now;
+    }
+
+    /// Increases current usage value.
+    ///
+    /// It is possible to send message even if over quota, e.g. if the message sending is initiated
+    /// by the user and should not be rate limited. However, sending messages when over quota
+    /// further postpones the time when it will be allowed to send low priority messages.
+    pub fn send(&mut self) {
+        self.send_at(SystemTime::now())
+    }
+
+    fn until_can_send_at(&self, now: SystemTime) -> Duration {
+        let current_value = self.current_value_at(now);
+        if current_value + 1.0 <= self.quota {
+            Duration::ZERO
+        } else {
+            let requirement = current_value + 1.0 - self.quota;
+            let rate = self.quota / self.window.as_secs_f64();
+            Duration::from_secs_f64(requirement / rate)
+        }
+    }
+
+    /// Calculates the time until `can_send` will return `true`.
+    pub fn until_can_send(&self) -> Duration {
+        self.until_can_send_at(SystemTime::now())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ratelimit() {
+        let now = SystemTime::now();
+
+        let mut ratelimit = Ratelimit::new_at(Duration::new(60, 0), 3.0, now);
+        assert!(ratelimit.can_send_at(now));
+
+        // Send burst of 3 messages.
+        ratelimit.send_at(now);
+        assert!(ratelimit.can_send_at(now));
+        ratelimit.send_at(now);
+        assert!(ratelimit.can_send_at(now));
+        ratelimit.send_at(now);
+
+        // Can't send more messages now.
+        assert!(!ratelimit.can_send_at(now));
+
+        // Can send one more message 20 seconds later.
+        assert_eq!(ratelimit.until_can_send_at(now), Duration::from_secs(20));
+        let now = now + Duration::from_secs(20);
+        assert!(ratelimit.can_send_at(now));
+        ratelimit.send_at(now);
+        assert!(!ratelimit.can_send_at(now));
+
+        // Send one more message anyway, over quota.
+        ratelimit.send_at(now);
+
+        // Always can send another message after 20 seconds,
+        // leaky bucket never overflows.
+        let now = now + Duration::from_secs(20);
+        assert!(ratelimit.can_send_at(now));
+
+        // Test that we don't panic if time appears to move backwards
+        assert!(!ratelimit.can_send_at(now - Duration::from_secs(20)));
+    }
+}

deltachat-repl/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "deltachat-repl"
+version = "1.111.0"
+license = "MPL-2.0"
+edition = "2021"
+
+[dependencies]
+ansi_term = "0.12.1"
+anyhow = "1"
+deltachat = { path = "..", features = ["internals"]}
+dirs = "4"
+log = "0.4.16"
+pretty_env_logger = "0.4"
+rusqlite = "0.28"
+rustyline = "11"
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+
+[features]
+default = ["vendored"]
+vendored = ["deltachat/vendored"]

deltachat-repl/src/cmdline.rs

@@ -1,9 +1,11 @@
+#![allow(clippy::format_push_string)]
 extern crate dirs;
 
+use std::path::Path;
 use std::str::FromStr;
+use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, ensure, Result};
-use async_std::path::Path;
 use deltachat::chat::{
     self, Chat, ChatId, ChatItem, ChatVisibility, MuteDuration, ProtectionStatus,
 };
@@ -11,30 +13,29 @@ use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::dc_receive_imf::*;
-use deltachat::dc_tools::*;
 use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
+use deltachat::reaction::send_reaction;
+use deltachat::receive_imf::*;
 use deltachat::sql;
-use deltachat::EventType;
+use deltachat::tools::*;
 use deltachat::{config, provider};
-use std::fs;
-use std::time::{Duration, SystemTime};
+use tokio::fs;
 
 /// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
-/// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
+/// e.g. bitmask 7 triggers actions defined with bits 1, 2 and 4.
 async fn reset_tables(context: &Context, bits: i32) {
-    println!("Resetting tables ({})...", bits);
+    println!("Resetting tables ({bits})...");
     if 0 != bits & 1 {
         context
             .sql()
-            .execute("DELETE FROM jobs;", paramsv![])
+            .execute("DELETE FROM jobs;", ())
             .await
             .unwrap();
         println!("(1) Jobs reset.");
@@ -42,7 +43,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 2 {
         context
             .sql()
-            .execute("DELETE FROM acpeerstates;", paramsv![])
+            .execute("DELETE FROM acpeerstates;", ())
             .await
             .unwrap();
         println!("(2) Peerstates reset.");
@@ -50,7 +51,7 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 4 {
         context
             .sql()
-            .execute("DELETE FROM keypairs;", paramsv![])
+            .execute("DELETE FROM keypairs;", ())
             .await
             .unwrap();
         println!("(4) Private keypairs reset.");
@@ -58,51 +59,49 @@ async fn reset_tables(context: &Context, bits: i32) {
     if 0 != bits & 8 {
         context
             .sql()
-            .execute("DELETE FROM contacts WHERE id>9;", paramsv![])
+            .execute("DELETE FROM contacts WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats WHERE id>9;", paramsv![])
+            .execute("DELETE FROM chats WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM chats_contacts;", paramsv![])
+            .execute("DELETE FROM chats_contacts;", ())
             .await
             .unwrap();
         context
             .sql()
-            .execute("DELETE FROM msgs WHERE id>9;", paramsv![])
+            .execute("DELETE FROM msgs WHERE id>9;", ())
             .await
             .unwrap();
         context
             .sql()
             .execute(
                 "DELETE FROM config WHERE keyname LIKE 'imap.%' OR keyname LIKE 'configured%';",
-                paramsv![],
+                (),
             )
             .await
             .unwrap();
+        context.sql().config_cache().write().await.clear();
         context
             .sql()
-            .execute("DELETE FROM leftgrps;", paramsv![])
+            .execute("DELETE FROM leftgrps;", ())
             .await
             .unwrap();
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(EventType::MsgsChanged {
-        chat_id: ChatId::new(0),
-        msg_id: MsgId::new(0),
-    });
+    context.emit_msgs_changed_without_ids();
 }
 
 async fn poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<()> {
-    let data = dc_read_file(context, filename).await?;
+    let data = read_file(context, filename).await?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", false).await {
-        println!("dc_receive_imf errored: {:?}", err);
+    if let Err(err) = receive_imf(context, &data, false).await {
+        println!("receive_imf errored: {err:?}");
     }
     Ok(())
 }
@@ -131,29 +130,25 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
         }
         real_spec = rs.unwrap();
     }
-    if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
+    if let Some(suffix) = get_filesuffix_lc(&real_spec) {
         if suffix == "eml" && poke_eml_file(context, &real_spec).await.is_ok() {
             read_cnt += 1
         }
     } else {
         /* import a directory */
         let dir_name = std::path::Path::new(&real_spec);
-        let dir = std::fs::read_dir(dir_name);
+        let dir = fs::read_dir(dir_name).await;
         if dir.is_err() {
             error!(context, "Import: Cannot open directory \"{}\".", &real_spec,);
             return false;
         } else {
-            let dir = dir.unwrap();
-            for entry in dir {
-                if entry.is_err() {
-                    break;
-                }
-                let entry = entry.unwrap();
+            let mut dir = dir.unwrap();
+            while let Ok(Some(entry)) = dir.next_entry().await {
                 let name_f = entry.file_name();
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
-                    println!("Import: {}", path_plus_name);
+                    println!("Import: {path_plus_name}");
                     if poke_eml_file(context, path_plus_name).await.is_ok() {
                         read_cnt += 1
                     }
@@ -163,10 +158,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: ChatId::new(0),
-            msg_id: MsgId::new(0),
-        });
+        context.emit_msgs_changed_without_ids();
     }
     true
 }
@@ -176,7 +168,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         .await
         .expect("invalid contact");
     let contact_name = if let Some(name) = msg.get_override_sender_name() {
-        format!("~{}", name)
+        format!("~{name}")
     } else {
         contact.get_display_name().to_string()
     };
@@ -197,7 +189,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         DownloadState::Failure => " [⬇ Download failed]",
     };
 
-    let temp2 = dc_timestamp_to_str(msg.get_timestamp());
+    let temp2 = timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
         "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
@@ -209,7 +201,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         contact_id,
         msgtext.unwrap_or_default(),
         if msg.has_html() { "[HAS-HTML]️" } else { "" },
-        if msg.get_from_id() == 1 {
+        if msg.get_from_id() == ContactId::SELF {
             ""
         } else if msg.get_state() == MessageState::InSeen {
             "[SEEN]"
@@ -225,6 +217,14 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
                 msg.get_videochat_url().unwrap_or_default(),
                 msg.get_videochat_type().unwrap_or_default()
             )
+        } else if msg.get_viewtype() == Viewtype::Webxdc {
+            match msg.get_webxdc_info(context).await {
+                Ok(info) => format!(
+                    "[WEBXDC: {}, icon={}, document={}, summary={}, source_code_url={}]",
+                    info.name, info.icon, info.document, info.summary, info.source_code_url
+                ),
+                Err(err) => format!("[get_webxdc_info() failed: {err}]"),
+            }
         } else {
             "".to_string()
         },
@@ -267,9 +267,8 @@ async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
+async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()> {
     for contact_id in contacts {
-        let line;
         let mut line2 = "".to_string();
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
@@ -284,24 +283,20 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) -> Result<()> {
         } else {
             ""
         };
-        line = format!(
+        let line = format!(
             "{}{} <{}>",
             if !name.is_empty() {
-                &name
+                name
             } else {
                 "<name unset>"
             },
             verified_str,
-            if !addr.is_empty() {
-                &addr
-            } else {
-                "addr unset"
-            }
+            if !addr.is_empty() { addr } else { "addr unset" }
         );
-        let peerstate = Peerstate::from_addr(context, &addr)
+        let peerstate = Peerstate::from_addr(context, addr)
             .await
             .expect("peerstate error");
-        if peerstate.is_some() && *contact_id != 1 {
+        if peerstate.is_some() && *contact_id != ContactId::SELF {
             line2 = format!(
                 ", prefer-encrypt={}",
                 peerstate.as_ref().unwrap().prefer_encrypt
@@ -410,8 +405,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
+                 resend <msg-id>\n\
                  markseen <msg-id>\n\
                  delmsg <msg-id>\n\
+                 react <msg-id> [<reaction>]\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
                  listverified [<query>]\n\
@@ -430,7 +427,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  joinqr <qr-content>\n\
                  setqr <qr-content>\n\
                  providerinfo <addr>\n\
-                 event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
                  clear -- clear screen\n\
@@ -439,10 +435,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ),
         },
         "initiate-key-transfer" => match initiate_key_transfer(&context).await {
-            Ok(setup_code) => println!(
-                "Setup code for the transferred setup message: {}",
-                setup_code,
-            ),
+            Ok(setup_code) => {
+                println!("Setup code for the transferred setup message: {setup_code}",)
+            }
             Err(err) => bail!("Failed to generate setup code: {}", err),
         },
         "get-setupcodebegin" => {
@@ -503,7 +498,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let setup_code = create_setup_code(&context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
             let file_content = render_setup_file(&context, &setup_code).await?;
-            async_std::fs::write(&file_name, file_content).await?;
+            fs::write(&file_name, file_content).await?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
                 file_name.display(),
@@ -532,7 +527,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
             let key = config::Config::from_str(arg1)?;
             let val = context.get_config(key).await;
-            println!("{}={:?}", key, val);
+            println!("{key}={val:?}");
         }
         "info" => {
             println!("{:#?}", context.get_info().await);
@@ -543,8 +538,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .join("connectivity.html");
             match context.get_connectivity_html().await {
                 Ok(html) => {
-                    fs::write(&file, html)?;
-                    println!("Report written to: {:#?}", file);
+                    fs::write(&file, html).await?;
+                    println!("Report written to: {file:#?}");
                 }
                 Err(err) => {
                     bail!("Failed to get connectivity html: {}", err);
@@ -558,7 +553,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             sql::housekeeping(&context).await.ok_or_log(&context);
         }
         "listchats" | "listarchived" | "chats" => {
-            let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let listflags = if arg0 == "listarchived" {
+                DC_GCL_ARCHIVED_ONLY
+            } else {
+                0
+            };
             let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
@@ -608,12 +607,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(summary.timestamp);
+                    let timestr = timestamp_to_str(summary.timestamp);
                     println!(
                         "{}{}{} [{}]{}",
                         summary
                             .prefix
-                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
+                            .map_or_else(String::new, |prefix| format!("{prefix}: ")),
                         summary.text,
                         statestr,
                         &timestr,
@@ -631,8 +630,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             if location::is_sending_locations_to_chat(&context, None).await? {
                 println!("Location streaming enabled.");
             }
-            println!("{} chats", cnt);
-            println!("{:?} to create this list", time_needed);
+            println!("{cnt} chats");
+            println!("{time_needed:?} to create this list");
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -640,7 +639,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             if !arg1.is_empty() {
                 let id = ChatId::new(arg1.parse()?);
-                println!("Selecting chat {}", id);
+                println!("Selecting chat {id}");
                 sel_chat = Some(Chat::load_from_db(&context, id).await?);
                 *chat_id = id;
             }
@@ -649,15 +648,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let time_start = std::time::SystemTime::now();
-            let msglist =
-                chat::get_chat_msgs(&context, sel_chat.get_id(), DC_GCM_ADDDAYMARKER, None).await?;
+            let msglist = chat::get_chat_msgs_ex(
+                &context,
+                sel_chat.get_id(),
+                chat::MessageListOptions {
+                    info_only: false,
+                    add_daymarker: true,
+                },
+            )
+            .await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             let msglist: Vec<MsgId> = msglist
                 .into_iter()
                 .map(|x| match x {
                     ChatItem::Message { msg_id } => msg_id,
-                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
                     ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
                 })
                 .collect();
@@ -687,7 +692,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 },
                 match sel_chat.get_profile_image(&context).await? {
                     Some(icon) => match icon.to_str() {
-                        Some(icon) => format!(" Icon: {}", icon),
+                        Some(icon) => format!(" Icon: {icon}"),
                         _ => " Icon: Err".to_string(),
                     },
                     _ => "".to_string(),
@@ -713,41 +718,40 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let time_noticed_needed = time_noticed_start.elapsed().unwrap_or_default();
 
             println!(
-                "{:?} to create this list, {:?} to mark all messages as noticed.",
-                time_needed, time_noticed_needed
+                "{time_needed:?} to create this list, {time_noticed_needed:?} to mark all messages as noticed."
             );
         }
         "createchat" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let chat_id = ChatId::create_for_contact(&context, contact_id).await?;
 
-            println!("Single#{} created successfully.", chat_id,);
+            println!("Single#{chat_id} created successfully.",);
         }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
                 chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
-            println!("Group#{} created successfully.", chat_id);
+            println!("Group#{chat_id} created successfully.");
         }
         "createbroadcast" => {
             let chat_id = chat::create_broadcast_list(&context).await?;
 
-            println!("Broadcast#{} created successfully.", chat_id);
+            println!("Broadcast#{chat_id} created successfully.");
         }
         "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
                 chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("Group#{} created and protected successfully.", chat_id);
+            println!("Group#{chat_id} created and protected successfully.");
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id_0: u32 = arg1.parse()?;
+            let contact_id_0 = ContactId::new(arg1.parse()?);
             chat::add_contact_to_chat(&context, sel_chat.as_ref().unwrap().get_id(), contact_id_0)
                 .await?;
             println!("Contact added to chat.");
@@ -755,7 +759,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "removemember" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id_1: u32 = arg1.parse()?;
+            let contact_id_1 = ContactId::new(arg1.parse()?);
             chat::remove_contact_from_chat(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
@@ -771,7 +775,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::set_chat_name(
                 &context,
                 sel_chat.as_ref().unwrap().get_id(),
-                &format!("{} {}", arg1, arg2).trim(),
+                format!("{arg1} {arg2}").trim(),
             )
             .await?;
 
@@ -822,7 +826,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!(
                     "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
-                    dc_timestamp_to_str(location.timestamp),
+                    timestamp_to_str(location.timestamp),
                     location.latitude,
                     location.longitude,
                     location.accuracy,
@@ -865,7 +869,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
-                println!("Success, streaming can be stoppped.");
+                println!("Success, streaming can be stopped.");
             }
         }
         "dellocations" => {
@@ -875,7 +879,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No message text given.");
 
-            let msg = format!("{} {}", arg1, arg2);
+            let msg = format!("{arg1} {arg2}");
 
             chat::send_text_msg(&context, sel_chat.as_ref().unwrap().get_id(), msg).await?;
         }
@@ -904,7 +908,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
             ensure!(!arg1.is_empty(), "No html-file given.");
             let path: &Path = arg1.as_ref();
-            let html = &*fs::read(&path)?;
+            let html = &*fs::read(&path).await?;
             let html = String::from_utf8_lossy(html);
 
             let mut msg = Message::new(Viewtype::Text);
@@ -917,7 +921,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
         "sendsyncmsg" => match context.send_sync_msg().await? {
-            Some(msg_id) => println!("sync message sent as {}.", msg_id),
+            Some(msg_id) => println!("sync message sent as {msg_id}."),
             None => println!("sync message not needed."),
         },
         "sendupdate" => {
@@ -937,14 +941,25 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
+            let query = format!("{arg1} {arg2}").trim().to_string();
             let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
             let time_start = std::time::SystemTime::now();
-            let msglist = context.search_msgs(chat, arg1).await?;
+            let msglist = context.search_msgs(chat, &query).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
 
             log_msglist(&context, &msglist).await?;
-            println!("{} messages.", msglist.len());
-            println!("{:?} to create this list", time_needed);
+            println!(
+                "{}{} messages for {}search of \"{}\"",
+                msglist.len(),
+                if msglist.len() == 1000 { "+" } else { "" },
+                if chat.is_none() {
+                    "global "
+                } else {
+                    "in-chat-"
+                },
+                query,
+            );
+            println!("{time_needed:?} to create this list");
         }
         "draft" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -979,11 +994,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "listmedia" => {
-            ensure!(sel_chat.is_some(), "No chat selected.");
-
             let images = chat::get_chat_media(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                sel_chat.map(|c| c.id),
                 Viewtype::Image,
                 Viewtype::Gif,
                 Viewtype::Video,
@@ -992,9 +1005,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
-                    print!("{}", data);
+                    print!("{data}");
                 } else {
-                    print!(", {}", data);
+                    print!(", {data}");
                 }
             }
             println!();
@@ -1065,12 +1078,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(&context, id).await?;
-            println!("{}", res);
+            println!("{res}");
         }
         "download" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
-            println!("Scheduling download for {:?}", id);
+            println!("Scheduling download for {id:?}");
             id.download_full(&context).await?;
         }
         "html" => {
@@ -1080,8 +1093,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .unwrap_or_default()
                 .join(format!("msg-{}.html", id.to_u32()));
             let html = id.get_html(&context).await?.unwrap_or_default();
-            fs::write(&file, html)?;
-            println!("HTML written to: {:#?}", file);
+            fs::write(&file, html).await?;
+            println!("HTML written to: {file:#?}");
         }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await?;
@@ -1100,6 +1113,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(&context, &msg_ids, chat_id).await?;
         }
+        "resend" => {
+            ensure!(!arg1.is_empty(), "Arguments <msg-id> expected");
+
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::resend_msgs(&context, &msg_ids).await?;
+        }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut msg_ids = vec![MsgId::new(0)];
@@ -1112,6 +1132,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ids[0] = MsgId::new(arg1.parse()?);
             message::delete_msgs(&context, &ids).await?;
         }
+        "react" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let msg_id = MsgId::new(arg1.parse()?);
+            let reaction = arg2;
+            send_reaction(&context, msg_id, reaction).await?;
+        }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
                 &context,
@@ -1130,7 +1156,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Arguments [<name>] <addr> expected.");
 
             if !arg2.is_empty() {
-                let book = format!("{}\n{}", arg1, arg2);
+                let book = format!("{arg1}\n{arg2}");
                 Contact::add_address_book(&context, &book).await?;
             } else {
                 Contact::create(&context, "", arg1).await?;
@@ -1139,7 +1165,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "contactinfo" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
 
-            let contact_id: u32 = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             let contact = Contact::get_by_id(&context, contact_id).await?;
             let name_n_addr = contact.get_name_n_addr();
 
@@ -1157,10 +1183,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chatlist = Chatlist::try_load(&context, 0, None, Some(contact_id)).await?;
             let chatlist_cnt = chatlist.len();
             if chatlist_cnt > 0 {
-                res += &format!(
-                    "\n\n{} chats shared with Contact#{}: ",
-                    chatlist_cnt, contact_id,
-                );
+                res += &format!("\n\n{chatlist_cnt} chats shared with Contact#{contact_id}: ",);
                 for i in 0..chatlist_cnt {
                     if 0 != i {
                         res += ", ";
@@ -1170,20 +1193,20 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 }
             }
 
-            println!("{}", res);
+            println!("{res}");
         }
         "delcontact" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            Contact::delete(&context, arg1.parse()?).await?;
+            Contact::delete(&context, ContactId::new(arg1.parse()?)).await?;
         }
         "block" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::block(&context, contact_id).await?;
         }
         "unblock" => {
             ensure!(!arg1.is_empty(), "Argument <contact-id> missing.");
-            let contact_id = arg1.parse()?;
+            let contact_id = ContactId::new(arg1.parse()?);
             Contact::unblock(&context, contact_id).await?;
         }
         "listblocked" => {
@@ -1194,13 +1217,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "checkqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             let qr = check_qr(&context, arg1).await?;
-            println!("qr={:?}", qr);
+            println!("qr={qr:?}");
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
             match set_config_from_qr(&context, arg1).await {
                 Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
-                Err(err) => println!("Cannot set config from QR code: {:?}", err),
+                Err(err) => println!("Cannot set config from QR code: {err:?}"),
             }
         }
         "providerinfo" => {
@@ -1210,7 +1233,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 .await?;
             match provider::get_provider_info(&context, arg1, socks5_enabled).await {
                 Some(info) => {
-                    println!("Information for provider belonging to {}:", arg1);
+                    println!("Information for provider belonging to {arg1}:");
                     println!("status: {}", info.status as u32);
                     println!("before_login_hint: {}", info.before_login_hint);
                     println!("after_login_hint: {}", info.after_login_hint);
@@ -1220,27 +1243,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     }
                 }
                 None => {
-                    println!("No information for provider belonging to {} found.", arg1);
+                    println!("No information for provider belonging to {arg1} found.");
                 }
             }
         }
-        // TODO: implement this again, unclear how to match this through though, without writing a parser.
-        // "event" => {
-        //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
-        //     let event = arg1.parse()?;
-        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
-        //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
-        //     println!(
-        //         "Sending event {:?}({}), received value {}.",
-        //         event, event as usize, r,
-        //     );
-        // }
         "fileinfo" => {
             ensure!(!arg1.is_empty(), "Argument <file> missing.");
 
-            if let Ok(buf) = dc_read_file(&context, &arg1).await {
-                let (width, height) = dc_get_filemeta(&buf)?;
-                println!("width={}, height={}", width, height);
+            if let Ok(buf) = read_file(&context, &arg1).await {
+                let (width, height) = get_filemeta(&buf)?;
+                println!("width={width}, height={height}");
             } else {
                 bail!("Command failed.");
             }
@@ -1251,8 +1263,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let device_cnt = message::estimate_deletion_cnt(&context, false, seconds).await?;
             let server_cnt = message::estimate_deletion_cnt(&context, true, seconds).await?;
             println!(
-                "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
-                seconds, device_cnt, server_cnt
+                "estimated count of messages older than {seconds} seconds:\non device: {device_cnt}\non server: {server_cnt}"
             );
         }
         "" => (),

deltachat-repl/src/main.rs

@@ -9,20 +9,21 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
 use anyhow::{bail, Error};
-use async_std::path::Path;
 use deltachat::chat::ChatId;
 use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
+use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::stock_str::StockStrings;
+use deltachat::{EventType, Events};
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
-use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
@@ -30,11 +31,11 @@ use rustyline::validate::Validator;
 use rustyline::{
     Cmd, CompletionType, Config, Context as RustyContext, EditMode, Editor, Helper, KeyEvent,
 };
+use tokio::fs;
+use tokio::runtime::Handle;
 
 mod cmdline;
 use self::cmdline::*;
-use deltachat::qr_code_generator::get_securejoin_qr_svg;
-use std::fs;
 
 /// Event Handler
 fn receive_event(event: EventType) {
@@ -66,8 +67,19 @@ fn receive_event(event: EventType) {
             info!(
                 "{}",
                 yellow.paint(format!(
-                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
-                    chat_id, msg_id,
+                    "Received MSGS_CHANGED(chat_id={chat_id}, msg_id={msg_id})",
+                ))
+            );
+        }
+        EventType::ReactionsChanged {
+            chat_id,
+            msg_id,
+            contact_id,
+        } => {
+            info!(
+                "{}",
+                yellow.paint(format!(
+                    "Received REACTIONS_CHANGED(chat_id={chat_id}, msg_id={msg_id}, contact_id={contact_id})"
                 ))
             );
         }
@@ -77,7 +89,7 @@ fn receive_event(event: EventType) {
         EventType::LocationChanged(contact) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+                yellow.paint(format!("Received LOCATION_CHANGED(contact={contact:?})"))
             );
         }
         EventType::ConfigureProgress { progress, comment } => {
@@ -85,21 +97,20 @@ fn receive_event(event: EventType) {
                 info!(
                     "{}",
                     yellow.paint(format!(
-                        "Received CONFIGURE_PROGRESS({} ‰, {})",
-                        progress, comment
+                        "Received CONFIGURE_PROGRESS({progress} ‰, {comment})"
                     ))
                 );
             } else {
                 info!(
                     "{}",
-                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({progress} ‰)"))
                 );
             }
         }
         EventType::ImexProgress(progress) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+                yellow.paint(format!("Received IMEX_PROGRESS({progress} ‰)"))
             );
         }
         EventType::ImexFileWritten(file) => {
@@ -111,7 +122,7 @@ fn receive_event(event: EventType) {
         EventType::ChatModified(chat) => {
             info!(
                 "{}",
-                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
+                yellow.paint(format!("Received CHAT_MODIFIED({chat})"))
             );
         }
         _ => {
@@ -207,14 +218,16 @@ const CHAT_COMMANDS: [&str; 36] = [
     "accept",
     "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 9] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
+    "resend",
     "markseen",
     "delmsg",
     "download",
+    "react",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -227,13 +240,12 @@ const CONTACT_COMMANDS: [&str; 9] = [
     "unblock",
     "listblocked",
 ];
-const MISC_COMMANDS: [&str; 12] = [
+const MISC_COMMANDS: [&str; 11] = [
     "getqr",
     "getqrsvg",
     "getbadqr",
     "checkqr",
     "joinqr",
-    "event",
     "fileinfo",
     "clear",
     "exit",
@@ -298,10 +310,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]), 0, Events::new(), StockStrings::new()).await?;
 
     let events = context.get_event_emitter();
-    async_std::task::spawn(async move {
+    tokio::task::spawn(async move {
         while let Some(event) = events.recv().await {
             receive_event(event.typ);
         }
@@ -313,17 +325,17 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
-        .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::channel::bounded(100);
-    let input_loop = async_std::task::spawn_blocking(move || {
+
+    let ctx = context.clone();
+    let input_loop = tokio::task::spawn_blocking(move || {
         let h = DcHelper {
             completer: FilenameCompleter::new(),
             highlighter: MatchingBracketHighlighter::new(),
             hinter: HistoryHinter {},
         };
-        let mut rl = Editor::with_config(config);
+        let mut rl = Editor::with_config(config)?;
         rl.set_helper(Some(h));
         rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
         rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);
@@ -338,17 +350,31 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
             match readline {
                 Ok(line) => {
                     // TODO: ignore "set mail_pw"
-                    rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line)).unwrap();
+                    rl.add_history_entry(line.as_str())?;
+                    let should_continue = Handle::current().block_on(async {
+                        match handle_cmd(line.trim(), ctx.clone(), &mut selected_chat).await {
+                            Ok(ExitResult::Continue) => true,
+                            Ok(ExitResult::Exit) => {
+                                println!("Exiting ...");
+                                false
+                            }
+                            Err(err) => {
+                                println!("Error: {err:#}");
+                                true
+                            }
+                        }
+                    });
+
+                    if !should_continue {
+                        break;
+                    }
                 }
                 Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
                     println!("Exiting...");
-                    drop(reader_s);
                     break;
                 }
                 Err(err) => {
-                    println!("Error: {}", err);
-                    drop(reader_s);
+                    println!("Error: {err:#}");
                     break;
                 }
             }
@@ -359,15 +385,8 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         Ok::<_, Error>(())
     });
 
-    while let Ok(line) = reader_r.recv().await {
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
-        }
-    }
     context.stop_io().await;
-    input_loop.await?;
+    input_loop.await??;
 
     Ok(())
 }
@@ -400,7 +419,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -416,15 +435,15 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
-            let mut qr = dc_get_securejoin_qr(&ctx, group).await?;
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
+            let mut qr = get_securejoin_qr(&ctx, group).await?;
             if !qr.is_empty() {
                 if arg0 == "getbadqr" && qr.len() > 40 {
                     qr.replace_range(12..22, "0000000000")
                 }
-                println!("{}", qr);
+                println!("{qr}");
                 let output = Command::new("qrencode")
-                    .args(&["-t", "ansiutf8", qr.as_str(), "-o", "-"])
+                    .args(["-t", "ansiutf8", qr.as_str(), "-o", "-"])
                     .output()
                     .expect("failed to execute process");
                 io::stdout().write_all(&output.stdout).unwrap();
@@ -433,12 +452,12 @@ async fn handle_cmd(
         }
         "getqrsvg" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
+            let group = arg1.parse::<u32>().ok().map(ChatId::new);
             let file = dirs::home_dir().unwrap_or_default().join("qr.svg");
             match get_securejoin_qr_svg(&ctx, group).await {
                 Ok(svg) => {
-                    fs::write(&file, svg)?;
-                    println!("QR code svg written to: {:#?}", file);
+                    fs::write(&file, svg).await?;
+                    println!("QR code svg written to: {file:#?}");
                 }
                 Err(err) => {
                     bail!("Failed to get QR code svg: {}", err);
@@ -448,7 +467,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await?;
+                join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
@@ -458,11 +477,12 @@ async fn handle_cmd(
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+#[tokio::main]
+async fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    start(args).await?;
 
     Ok(())
 }

deltachat-rpc-client/README.md

@@ -0,0 +1,41 @@
+# Delta Chat RPC python client
+
+RPC client connects to standalone Delta Chat RPC server `deltachat-rpc-server`
+and provides asynchronous interface to it.
+
+## Getting started
+
+To use Delta Chat RPC client, first build a `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
+Install it anywhere in your `PATH`.
+
+## Testing
+
+1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
+2. Run `PATH="../target/debug:$PATH" tox`.
+
+Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
+
+## Using in REPL
+
+Setup a development environment:
+```
+$ tox --devenv env
+$ . env/bin/activate
+```
+
+It is recommended to use IPython, because it supports using `await` directly
+from the REPL.
+
+```
+$ pip install ipython
+$ PATH="../target/debug:$PATH" ipython
+...
+In [1]: from deltachat_rpc_client import *
+In [2]: rpc = Rpc()
+In [3]: await rpc.start()
+In [4]: dc = DeltaChat(rpc)
+In [5]: system_info = await dc.get_system_info()
+In [6]: system_info["level"]
+Out[6]: 'awesome'
+In [7]: await rpc.close()
+```

deltachat-rpc-client/examples/echobot.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+"""Minimal echo bot example.
+
+it will echo back any text send to it, it also will print to console all Delta Chat core events.
+Pass --help to the CLI to see available options.
+"""
+import asyncio
+
+from deltachat_rpc_client import events, run_bot_cli
+
+hooks = events.HookCollection()
+
+
+@hooks.on(events.RawEvent)
+async def log_event(event):
+    print(event)
+
+
+@hooks.on(events.NewMessage)
+async def echo(event):
+    snapshot = event.message_snapshot
+    await snapshot.chat.send_text(snapshot.text)
+
+
+if __name__ == "__main__":
+    asyncio.run(run_bot_cli(hooks))

deltachat-rpc-client/examples/echobot_advanced.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Advanced echo bot example.
+
+it will echo back any message that has non-empty text and also supports the /help command.
+"""
+import asyncio
+import logging
+import sys
+
+from deltachat_rpc_client import Bot, DeltaChat, EventType, Rpc, events
+
+hooks = events.HookCollection()
+
+
+@hooks.on(events.RawEvent)
+async def log_event(event):
+    if event.type == EventType.INFO:
+        logging.info(event.msg)
+    elif event.type == EventType.WARNING:
+        logging.warning(event.msg)
+
+
+@hooks.on(events.RawEvent(EventType.ERROR))
+async def log_error(event):
+    logging.error(event.msg)
+
+
+@hooks.on(events.MemberListChanged)
+async def on_memberlist_changed(event):
+    logging.info("member %s was %s", event.member, "added" if event.member_added else "removed")
+
+
+@hooks.on(events.GroupImageChanged)
+async def on_group_image_changed(event):
+    logging.info("group image %s", "deleted" if event.image_deleted else "changed")
+
+
+@hooks.on(events.GroupNameChanged)
+async def on_group_name_changed(event):
+    logging.info("group name changed, old name: %s", event.old_name)
+
+
+@hooks.on(events.NewMessage(func=lambda e: not e.command))
+async def echo(event):
+    snapshot = event.message_snapshot
+    if snapshot.text or snapshot.file:
+        await snapshot.chat.send_message(text=snapshot.text, file=snapshot.file)
+
+
+@hooks.on(events.NewMessage(command="/help"))
+async def help_command(event):
+    snapshot = event.message_snapshot
+    await snapshot.chat.send_text("Send me any message and I will echo it back")
+
+
+async def main():
+    async with Rpc() as rpc:
+        deltachat = DeltaChat(rpc)
+        system_info = await deltachat.get_system_info()
+        logging.info("Running deltachat core %s", system_info.deltachat_core_version)
+
+        accounts = await deltachat.get_all_accounts()
+        account = accounts[0] if accounts else await deltachat.add_account()
+
+        bot = Bot(account, hooks)
+        if not await bot.is_configured():
+            # Save a reference to avoid garbage collection of the task.
+            _configure_task = asyncio.create_task(bot.configure(email=sys.argv[1], password=sys.argv[2]))
+        await bot.run_forever()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    asyncio.run(main())

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""
+Example echo bot without using hooks
+"""
+import asyncio
+import logging
+import sys
+
+from deltachat_rpc_client import DeltaChat, EventType, Rpc
+
+
+async def main():
+    async with Rpc() as rpc:
+        deltachat = DeltaChat(rpc)
+        system_info = await deltachat.get_system_info()
+        logging.info("Running deltachat core %s", system_info["deltachat_core_version"])
+
+        accounts = await deltachat.get_all_accounts()
+        account = accounts[0] if accounts else await deltachat.add_account()
+
+        await account.set_config("bot", "1")
+        if not await account.is_configured():
+            logging.info("Account is not configured, configuring")
+            await account.set_config("addr", sys.argv[1])
+            await account.set_config("mail_pw", sys.argv[2])
+            await account.configure()
+            logging.info("Configured")
+        else:
+            logging.info("Account is already configured")
+            await deltachat.start_io()
+
+        async def process_messages():
+            for message in await account.get_fresh_messages_in_arrival_order():
+                snapshot = await message.get_snapshot()
+                if not snapshot.is_bot and not snapshot.is_info:
+                    await snapshot.chat.send_text(snapshot.text)
+                await snapshot.message.mark_seen()
+
+        # Process old messages.
+        await process_messages()
+
+        while True:
+            event = await account.wait_for_event()
+            if event["type"] == EventType.INFO:
+                logging.info("%s", event["msg"])
+            elif event["type"] == EventType.WARNING:
+                logging.warning("%s", event["msg"])
+            elif event["type"] == EventType.ERROR:
+                logging.error("%s", event["msg"])
+            elif event["type"] == EventType.INCOMING_MSG:
+                logging.info("Got an incoming message")
+                await process_messages()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    asyncio.run(main())

deltachat-rpc-client/pyproject.toml

@@ -0,0 +1,60 @@
+[build-system]
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deltachat-rpc-client"
+description = "Python client for Delta Chat core JSON-RPC interface"
+dependencies = [
+    "aiohttp",
+    "aiodns"
+]
+dynamic = [
+    "version"
+]
+
+[tool.setuptools.package-data]
+deltachat_rpc_client = [
+    "py.typed"
+]
+
+[project.entry-points.pytest11]
+"deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin"
+
+[tool.black]
+line-length = 120
+
+[tool.ruff]
+select = [
+  "E", "W", # pycodestyle
+  "F", # Pyflakes
+  "N", # pep8-naming
+  "I", # isort
+
+  "ARG", # flake8-unused-arguments
+  "B", # flake8-bugbear
+  "C4", # flake8-comprehensions
+  "COM", # flake8-commas
+  "DTZ", # flake8-datetimez
+  "ICN", # flake8-import-conventions
+  "ISC", # flake8-implicit-str-concat
+  "PIE", # flake8-pie
+  "PT", # flake8-pytest-style
+  "RET", # flake8-return
+  "SIM", # flake8-simplify
+  "TCH", # flake8-type-checking
+  "TID", # flake8-tidy-imports
+  "YTT", # flake8-2020
+
+  "ERA", # eradicate
+
+  "PLC", # Pylint Convention
+  "PLE", # Pylint Error
+  "PLW", # Pylint Warning
+
+  "RUF006" # asyncio-dangling-task
+]
+line-length = 120
+
+[tool.isort]
+profile = "black"

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -0,0 +1,25 @@
+"""Delta Chat asynchronous high-level API"""
+from ._utils import AttrDict, run_bot_cli, run_client_cli
+from .account import Account
+from .chat import Chat
+from .client import Bot, Client
+from .const import EventType
+from .contact import Contact
+from .deltachat import DeltaChat
+from .message import Message
+from .rpc import Rpc
+
+__all__ = [
+    "Account",
+    "AttrDict",
+    "Bot",
+    "Chat",
+    "Client",
+    "Contact",
+    "DeltaChat",
+    "EventType",
+    "Message",
+    "Rpc",
+    "run_bot_cli",
+    "run_client_cli",
+]

deltachat-rpc-client/src/deltachat_rpc_client/_utils.py

@@ -0,0 +1,170 @@
+import argparse
+import asyncio
+import re
+import sys
+from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple, Type, Union
+
+if TYPE_CHECKING:
+    from .client import Client
+    from .events import EventFilter
+
+
+def _camel_to_snake(name: str) -> str:
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    name = re.sub("__([A-Z])", r"_\1", name)
+    name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", name)
+    return name.lower()
+
+
+def _to_attrdict(obj):
+    if isinstance(obj, AttrDict):
+        return obj
+    if isinstance(obj, dict):
+        return AttrDict(obj)
+    if isinstance(obj, list):
+        return [_to_attrdict(elem) for elem in obj]
+    return obj
+
+
+class AttrDict(dict):
+    """Dictionary that allows accessing values using the "dot notation" as attributes."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__({_camel_to_snake(key): _to_attrdict(value) for key, value in dict(*args, **kwargs).items()})
+
+    def __getattr__(self, attr):
+        if attr in self:
+            return self[attr]
+        raise AttributeError("Attribute not found: " + str(attr))
+
+    def __setattr__(self, attr, val):
+        if attr in self:
+            raise AttributeError("Attribute-style access is read only")
+        super().__setattr__(attr, val)
+
+
+async def run_client_cli(
+    hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
+    argv: Optional[list] = None,
+    **kwargs,
+) -> None:
+    """Run a simple command line app, using the given hooks.
+
+    Extra keyword arguments are passed to the internal Rpc object.
+    """
+    from .client import Client
+
+    await _run_cli(Client, hooks, argv, **kwargs)
+
+
+async def run_bot_cli(
+    hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
+    argv: Optional[list] = None,
+    **kwargs,
+) -> None:
+    """Run a simple bot command line using the given hooks.
+
+    Extra keyword arguments are passed to the internal Rpc object.
+    """
+    from .client import Bot
+
+    await _run_cli(Bot, hooks, argv, **kwargs)
+
+
+async def _run_cli(
+    client_type: Type["Client"],
+    hooks: Optional[Iterable[Tuple[Callable, Union[type, "EventFilter"]]]] = None,
+    argv: Optional[list] = None,
+    **kwargs,
+) -> None:
+    from .deltachat import DeltaChat
+    from .rpc import Rpc
+
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
+    parser.add_argument(
+        "accounts_dir",
+        help="accounts folder (default: current working directory)",
+        nargs="?",
+    )
+    parser.add_argument("--email", action="store", help="email address")
+    parser.add_argument("--password", action="store", help="password")
+    args = parser.parse_args(argv[1:])
+
+    async with Rpc(accounts_dir=args.accounts_dir, **kwargs) as rpc:
+        deltachat = DeltaChat(rpc)
+        core_version = (await deltachat.get_system_info()).deltachat_core_version
+        accounts = await deltachat.get_all_accounts()
+        account = accounts[0] if accounts else await deltachat.add_account()
+
+        client = client_type(account, hooks)
+        client.logger.debug("Running deltachat core %s", core_version)
+        if not await client.is_configured():
+            assert args.email, "Account is not configured and email must be provided"
+            assert args.password, "Account is not configured and password must be provided"
+            # Save a reference to avoid garbage collection of the task.
+            _configure_task = asyncio.create_task(client.configure(email=args.email, password=args.password))
+        await client.run_forever()
+
+
+def extract_addr(text: str) -> str:
+    """extract email address from the given text."""
+    match = re.match(r".*\((.+@.+)\)", text)
+    if match:
+        text = match.group(1)
+    text = text.rstrip(".")
+    return text.strip()
+
+
+def parse_system_image_changed(text: str) -> Optional[Tuple[str, bool]]:
+    """return image changed/deleted info from parsing the given system message text."""
+    text = text.lower()
+    match = re.match(r"group image (changed|deleted) by (.+).", text)
+    if match:
+        action, actor = match.groups()
+        return (extract_addr(actor), action == "deleted")
+    return None
+
+
+def parse_system_title_changed(text: str) -> Optional[Tuple[str, str]]:
+    text = text.lower()
+    match = re.match(r'group name changed from "(.+)" to ".+" by (.+).', text)
+    if match:
+        old_title, actor = match.groups()
+        return (extract_addr(actor), old_title)
+    return None
+
+
+def parse_system_add_remove(text: str) -> Optional[Tuple[str, str, str]]:
+    """return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) tuple.
+    """
+    # You removed member a@b.
+    # You added member a@b.
+    # Member Me (x@y) removed by a@b.
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
+    text = text.lower()
+
+    match = re.match(r"member (.+) (removed|added) by (.+)", text)
+    if match:
+        affected, action, actor = match.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+
+    match = re.match(r"you (removed|added) member (.+)", text)
+    if match:
+        action, affected = match.groups()
+        return action, extract_addr(affected), "me"
+
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr
+
+    return None

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -0,0 +1,245 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+
+from ._utils import AttrDict
+from .chat import Chat
+from .const import ChatlistFlag, ContactFlag, SpecialContactId
+from .contact import Contact
+from .message import Message
+
+if TYPE_CHECKING:
+    from .deltachat import DeltaChat
+    from .rpc import Rpc
+
+
+@dataclass
+class Account:
+    """Delta Chat account."""
+
+    manager: "DeltaChat"
+    id: int
+
+    @property
+    def _rpc(self) -> "Rpc":
+        return self.manager.rpc
+
+    async def wait_for_event(self) -> AttrDict:
+        """Wait until the next event and return it."""
+        return AttrDict(await self._rpc.wait_for_event(self.id))
+
+    async def remove(self) -> None:
+        """Remove the account."""
+        await self._rpc.remove_account(self.id)
+
+    async def start_io(self) -> None:
+        """Start the account I/O."""
+        await self._rpc.start_io(self.id)
+
+    async def stop_io(self) -> None:
+        """Stop the account I/O."""
+        await self._rpc.stop_io(self.id)
+
+    async def get_info(self) -> AttrDict:
+        """Return dictionary of this account configuration parameters."""
+        return AttrDict(await self._rpc.get_info(self.id))
+
+    async def get_size(self) -> int:
+        """Get the combined filesize of an account in bytes."""
+        return await self._rpc.get_account_file_size(self.id)
+
+    async def is_configured(self) -> bool:
+        """Return True if this account is configured."""
+        return await self._rpc.is_configured(self.id)
+
+    async def set_config(self, key: str, value: Optional[str] = None) -> None:
+        """Set configuration value."""
+        await self._rpc.set_config(self.id, key, value)
+
+    async def get_config(self, key: str) -> Optional[str]:
+        """Get configuration value."""
+        return await self._rpc.get_config(self.id, key)
+
+    async def update_config(self, **kwargs) -> None:
+        """update config values."""
+        for key, value in kwargs.items():
+            await self.set_config(key, value)
+
+    async def set_avatar(self, img_path: Optional[str] = None) -> None:
+        """Set self avatar.
+
+        Passing None will discard the currently set avatar.
+        """
+        await self.set_config("selfavatar", img_path)
+
+    async def get_avatar(self) -> Optional[str]:
+        """Get self avatar."""
+        return await self.get_config("selfavatar")
+
+    async def configure(self) -> None:
+        """Configure an account."""
+        await self._rpc.configure(self.id)
+
+    async def create_contact(self, obj: Union[int, str, Contact], name: Optional[str] = None) -> Contact:
+        """Create a new Contact or return an existing one.
+
+        Calling this method will always result in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        name is updated if specified.
+
+        :param obj: email-address or contact id.
+        :param name: (optional) display name for this contact.
+        """
+        if isinstance(obj, int):
+            obj = Contact(self, obj)
+        if isinstance(obj, Contact):
+            obj = (await obj.get_snapshot()).address
+        return Contact(self, await self._rpc.create_contact(self.id, obj, name))
+
+    def get_contact_by_id(self, contact_id: int) -> Contact:
+        """Return Contact instance for the given contact ID."""
+        return Contact(self, contact_id)
+
+    async def get_contact_by_addr(self, address: str) -> Optional[Contact]:
+        """Check if an e-mail address belongs to a known and unblocked contact."""
+        contact_id = await self._rpc.lookup_contact_id_by_addr(self.id, address)
+        return contact_id and Contact(self, contact_id)
+
+    async def get_blocked_contacts(self) -> List[AttrDict]:
+        """Return a list with snapshots of all blocked contacts."""
+        contacts = await self._rpc.get_blocked_contacts(self.id)
+        return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
+
+    async def get_contacts(
+        self,
+        query: Optional[str] = None,
+        with_self: bool = False,
+        verified_only: bool = False,
+        snapshot: bool = False,
+    ) -> Union[List[Contact], List[AttrDict]]:
+        """Get a filtered list of contacts.
+
+        :param query: if a string is specified, only return contacts
+                      whose name or e-mail matches query.
+        :param with_self: if True the self-contact is also included if it matches the query.
+        :param only_verified: if True only return verified contacts.
+        :param snapshot: If True return a list of contact snapshots instead of Contact instances.
+        """
+        flags = 0
+        if verified_only:
+            flags |= ContactFlag.VERIFIED_ONLY
+        if with_self:
+            flags |= ContactFlag.ADD_SELF
+
+        if snapshot:
+            contacts = await self._rpc.get_contacts(self.id, flags, query)
+            return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
+        contacts = await self._rpc.get_contact_ids(self.id, flags, query)
+        return [Contact(self, contact_id) for contact_id in contacts]
+
+    @property
+    def self_contact(self) -> Contact:
+        """This account's identity as a Contact."""
+        return Contact(self, SpecialContactId.SELF)
+
+    async def get_chatlist(
+        self,
+        query: Optional[str] = None,
+        contact: Optional[Contact] = None,
+        archived_only: bool = False,
+        for_forwarding: bool = False,
+        no_specials: bool = False,
+        alldone_hint: bool = False,
+        snapshot: bool = False,
+    ) -> Union[List[Chat], List[AttrDict]]:
+        """Return list of chats.
+
+        :param query: if a string is specified only chats matching this query are returned.
+        :param contact: if a contact is specified only chats including this contact are returned.
+        :param archived_only: if True only archived chats are returned.
+        :param for_forwarding: if True the chat list is sorted with "Saved messages" at the top
+                               and without "Device chat" and contact requests.
+        :param no_specials: if True archive link is not added to the list.
+        :param alldone_hint: if True the "all done hint" special chat will be added to the list
+                             as needed.
+        :param snapshot: If True return a list of chat snapshots instead of Chat instances.
+        """
+        flags = 0
+        if archived_only:
+            flags |= ChatlistFlag.ARCHIVED_ONLY
+        if for_forwarding:
+            flags |= ChatlistFlag.FOR_FORWARDING
+        if no_specials:
+            flags |= ChatlistFlag.NO_SPECIALS
+        if alldone_hint:
+            flags |= ChatlistFlag.ADD_ALLDONE_HINT
+
+        entries = await self._rpc.get_chatlist_entries(self.id, flags, query, contact and contact.id)
+        if not snapshot:
+            return [Chat(self, entry[0]) for entry in entries]
+
+        items = await self._rpc.get_chatlist_items_by_entries(self.id, entries)
+        chats = []
+        for item in items.values():
+            item["chat"] = Chat(self, item["id"])
+            chats.append(AttrDict(item))
+        return chats
+
+    async def create_group(self, name: str, protect: bool = False) -> Chat:
+        """Create a new group chat.
+
+        After creation, the group has only self-contact as member and is in unpromoted state.
+        """
+        return Chat(self, await self._rpc.create_group_chat(self.id, name, protect))
+
+    def get_chat_by_id(self, chat_id: int) -> Chat:
+        """Return the Chat instance with the given ID."""
+        return Chat(self, chat_id)
+
+    async def secure_join(self, qrdata: str) -> Chat:
+        """Continue a Setup-Contact or Verified-Group-Invite protocol started on
+        another device.
+
+        The function returns immediately and the handshake runs in background, sending
+        and receiving several messages.
+        Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
+        See https://countermitm.readthedocs.io/en/latest/new.html for protocol details.
+
+        :param qrdata: The text of the scanned QR code.
+        """
+        return Chat(self, await self._rpc.secure_join(self.id, qrdata))
+
+    async def get_qr_code(self) -> Tuple[str, str]:
+        """Get Setup-Contact QR Code text and SVG data.
+
+        this data needs to be transferred to another Delta Chat account
+        in a second channel, typically used by mobiles with QRcode-show + scan UX.
+        """
+        return await self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
+
+    def get_message_by_id(self, msg_id: int) -> Message:
+        """Return the Message instance with the given ID."""
+        return Message(self, msg_id)
+
+    async def mark_seen_messages(self, messages: List[Message]) -> None:
+        """Mark the given set of messages as seen."""
+        await self._rpc.markseen_msgs(self.id, [msg.id for msg in messages])
+
+    async def delete_messages(self, messages: List[Message]) -> None:
+        """Delete messages (local and remote)."""
+        await self._rpc.delete_messages(self.id, [msg.id for msg in messages])
+
+    async def get_fresh_messages(self) -> List[Message]:
+        """Return the list of fresh messages, newest messages first.
+
+        This call is intended for displaying notifications.
+        If you are writing a bot, use `get_fresh_messages_in_arrival_order()` instead,
+        to process oldest messages first.
+        """
+        fresh_msg_ids = await self._rpc.get_fresh_msgs(self.id)
+        return [Message(self, msg_id) for msg_id in fresh_msg_ids]
+
+    async def get_fresh_messages_in_arrival_order(self) -> List[Message]:
+        """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
+        fresh_msg_ids = sorted(await self._rpc.get_fresh_msgs(self.id))
+        return [Message(self, msg_id) for msg_id in fresh_msg_ids]

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -0,0 +1,254 @@
+import calendar
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
+
+from ._utils import AttrDict
+from .const import ChatVisibility, ViewType
+from .contact import Contact
+from .message import Message
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from .account import Account
+    from .rpc import Rpc
+
+
+@dataclass
+class Chat:
+    """Chat object which manages members and through which you can send and retrieve messages."""
+
+    account: "Account"
+    id: int
+
+    @property
+    def _rpc(self) -> "Rpc":
+        return self.account._rpc
+
+    async def delete(self) -> None:
+        """Delete this chat and all its messages.
+
+        Note:
+
+        - does not delete messages on server
+        - the chat or contact is not blocked, new message will arrive
+        """
+        await self._rpc.delete_chat(self.account.id, self.id)
+
+    async def block(self) -> None:
+        """Block this chat."""
+        await self._rpc.block_chat(self.account.id, self.id)
+
+    async def accept(self) -> None:
+        """Accept this contact request chat."""
+        await self._rpc.accept_chat(self.account.id, self.id)
+
+    async def leave(self) -> None:
+        """Leave this chat."""
+        await self._rpc.leave_group(self.account.id, self.id)
+
+    async def mute(self, duration: Optional[int] = None) -> None:
+        """Mute this chat, if a duration is not provided the chat is muted forever.
+
+        :param duration: mute duration from now in seconds. Must be greater than zero.
+        """
+        if duration is not None:
+            assert duration > 0, "Invalid duration"
+            dur: Union[str, dict] = {"Until": duration}
+        else:
+            dur = "Forever"
+        await self._rpc.set_chat_mute_duration(self.account.id, self.id, dur)
+
+    async def unmute(self) -> None:
+        """Unmute this chat."""
+        await self._rpc.set_chat_mute_duration(self.account.id, self.id, "NotMuted")
+
+    async def pin(self) -> None:
+        """Pin this chat."""
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.PINNED)
+
+    async def unpin(self) -> None:
+        """Unpin this chat."""
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL)
+
+    async def archive(self) -> None:
+        """Archive this chat."""
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.ARCHIVED)
+
+    async def unarchive(self) -> None:
+        """Unarchive this chat."""
+        await self._rpc.set_chat_visibility(self.account.id, self.id, ChatVisibility.NORMAL)
+
+    async def set_name(self, name: str) -> None:
+        """Set name of this chat."""
+        await self._rpc.set_chat_name(self.account.id, self.id, name)
+
+    async def set_ephemeral_timer(self, timer: int) -> None:
+        """Set ephemeral timer of this chat."""
+        await self._rpc.set_chat_ephemeral_timer(self.account.id, self.id, timer)
+
+    async def get_encryption_info(self) -> str:
+        """Return encryption info for this chat."""
+        return await self._rpc.get_chat_encryption_info(self.account.id, self.id)
+
+    async def get_qr_code(self) -> Tuple[str, str]:
+        """Get Join-Group QR code text and SVG data."""
+        return await self._rpc.get_chat_securejoin_qr_code_svg(self.account.id, self.id)
+
+    async def get_basic_snapshot(self) -> AttrDict:
+        """Get a chat snapshot with basic info about this chat."""
+        info = await self._rpc.get_basic_chat_info(self.account.id, self.id)
+        return AttrDict(chat=self, **info)
+
+    async def get_full_snapshot(self) -> AttrDict:
+        """Get a full snapshot of this chat."""
+        info = await self._rpc.get_full_chat_by_id(self.account.id, self.id)
+        return AttrDict(chat=self, **info)
+
+    async def send_message(
+        self,
+        text: Optional[str] = None,
+        html: Optional[str] = None,
+        viewtype: Optional[ViewType] = None,
+        file: Optional[str] = None,
+        location: Optional[Tuple[float, float]] = None,
+        override_sender_name: Optional[str] = None,
+        quoted_msg: Optional[Union[int, Message]] = None,
+    ) -> Message:
+        """Send a message and return the resulting Message instance."""
+        if isinstance(quoted_msg, Message):
+            quoted_msg = quoted_msg.id
+
+        draft = {
+            "text": text,
+            "html": html,
+            "viewtype": viewtype,
+            "file": file,
+            "location": location,
+            "overrideSenderName": override_sender_name,
+            "quotedMsg": quoted_msg,
+        }
+        msg_id = await self._rpc.send_msg(self.account.id, self.id, draft)
+        return Message(self.account, msg_id)
+
+    async def send_text(self, text: str) -> Message:
+        """Send a text message and return the resulting Message instance."""
+        msg_id = await self._rpc.misc_send_text_message(self.account.id, self.id, text)
+        return Message(self.account, msg_id)
+
+    async def send_videochat_invitation(self) -> Message:
+        """Send a videochat invitation and return the resulting Message instance."""
+        msg_id = await self._rpc.send_videochat_invitation(self.account.id, self.id)
+        return Message(self.account, msg_id)
+
+    async def send_sticker(self, path: str) -> Message:
+        """Send an sticker and return the resulting Message instance."""
+        msg_id = await self._rpc.send_sticker(self.account.id, self.id, path)
+        return Message(self.account, msg_id)
+
+    async def forward_messages(self, messages: List[Message]) -> None:
+        """Forward a list of messages to this chat."""
+        msg_ids = [msg.id for msg in messages]
+        await self._rpc.forward_messages(self.account.id, msg_ids, self.id)
+
+    async def set_draft(
+        self,
+        text: Optional[str] = None,
+        file: Optional[str] = None,
+        quoted_msg: Optional[int] = None,
+    ) -> None:
+        """Set draft message."""
+        if isinstance(quoted_msg, Message):
+            quoted_msg = quoted_msg.id
+        await self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg)
+
+    async def remove_draft(self) -> None:
+        """Remove draft message."""
+        await self._rpc.remove_draft(self.account.id, self.id)
+
+    async def get_draft(self) -> Optional[AttrDict]:
+        """Get draft message."""
+        snapshot = await self._rpc.get_draft(self.account.id, self.id)
+        if not snapshot:
+            return None
+        snapshot = AttrDict(snapshot)
+        snapshot["chat"] = Chat(self.account, snapshot.chat_id)
+        snapshot["sender"] = Contact(self.account, snapshot.from_id)
+        snapshot["message"] = Message(self.account, snapshot.id)
+        return snapshot
+
+    async def get_messages(self, info_only: bool = False, add_daymarker: bool = False) -> List[Message]:
+        """get the list of messages in this chat."""
+        msgs = await self._rpc.get_message_ids(self.account.id, self.id, info_only, add_daymarker)
+        return [Message(self.account, msg_id) for msg_id in msgs]
+
+    async def get_fresh_message_count(self) -> int:
+        """Get number of fresh messages in this chat"""
+        return await self._rpc.get_fresh_msg_cnt(self.account.id, self.id)
+
+    async def mark_noticed(self) -> None:
+        """Mark all messages in this chat as noticed."""
+        await self._rpc.marknoticed_chat(self.account.id, self.id)
+
+    async def add_contact(self, *contact: Union[int, str, Contact]) -> None:
+        """Add contacts to this group."""
+        for cnt in contact:
+            if isinstance(cnt, str):
+                contact_id = (await self.account.create_contact(cnt)).id
+            elif not isinstance(cnt, int):
+                contact_id = cnt.id
+            else:
+                contact_id = cnt
+            await self._rpc.add_contact_to_chat(self.account.id, self.id, contact_id)
+
+    async def remove_contact(self, *contact: Union[int, str, Contact]) -> None:
+        """Remove members from this group."""
+        for cnt in contact:
+            if isinstance(cnt, str):
+                contact_id = (await self.account.create_contact(cnt)).id
+            elif not isinstance(cnt, int):
+                contact_id = cnt.id
+            else:
+                contact_id = cnt
+            await self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id)
+
+    async def get_contacts(self) -> List[Contact]:
+        """Get the contacts belonging to this chat.
+
+        For single/direct chats self-address is not included.
+        """
+        contacts = await self._rpc.get_chat_contacts(self.account.id, self.id)
+        return [Contact(self.account, contact_id) for contact_id in contacts]
+
+    async def set_image(self, path: str) -> None:
+        """Set profile image of this chat.
+
+        :param path: Full path of the image to use as the group image.
+        """
+        await self._rpc.set_chat_profile_image(self.account.id, self.id, path)
+
+    async def remove_image(self) -> None:
+        """Remove profile image of this chat."""
+        await self._rpc.set_chat_profile_image(self.account.id, self.id, None)
+
+    async def get_locations(
+        self,
+        contact: Optional[Contact] = None,
+        timestamp_from: Optional["datetime"] = None,
+        timestamp_to: Optional["datetime"] = None,
+    ) -> List[AttrDict]:
+        """Get list of location snapshots for the given contact in the given timespan."""
+        time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0
+        time_to = calendar.timegm(timestamp_to.utctimetuple()) if timestamp_to else 0
+        contact_id = contact.id if contact else 0
+
+        result = await self._rpc.get_locations(self.account.id, self.id, contact_id, time_from, time_to)
+        locations = []
+        contacts: Dict[int, Contact] = {}
+        for loc in result:
+            location = AttrDict(loc)
+            location["chat"] = self
+            location["contact"] = contacts.setdefault(location.contact_id, Contact(self.account, location.contact_id))
+            location["message"] = Message(self.account, location.msg_id)
+            locations.append(location)
+        return locations

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -0,0 +1,205 @@
+"""Event loop implementations offering high level event handling/hooking."""
+import inspect
+import logging
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Coroutine,
+    Dict,
+    Iterable,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+)
+
+from ._utils import (
+    AttrDict,
+    parse_system_add_remove,
+    parse_system_image_changed,
+    parse_system_title_changed,
+)
+from .const import COMMAND_PREFIX, EventType, SystemMessageType
+from .events import (
+    EventFilter,
+    GroupImageChanged,
+    GroupNameChanged,
+    MemberListChanged,
+    NewMessage,
+    RawEvent,
+)
+
+if TYPE_CHECKING:
+    from deltachat_rpc_client.account import Account
+
+
+class Client:
+    """Simple Delta Chat client that listen to events of a single account."""
+
+    def __init__(
+        self,
+        account: "Account",
+        hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None,
+        logger: Optional[logging.Logger] = None,
+    ) -> None:
+        self.account = account
+        self.logger = logger or logging
+        self._hooks: Dict[type, Set[tuple]] = {}
+        self._should_process_messages = 0
+        self.add_hooks(hooks or [])
+
+    def add_hooks(self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]) -> None:
+        for hook, event in hooks:
+            self.add_hook(hook, event)
+
+    def add_hook(self, hook: Callable, event: Union[type, EventFilter] = RawEvent) -> None:
+        """Register hook for the given event filter."""
+        if isinstance(event, type):
+            event = event()
+        assert isinstance(event, EventFilter)
+        self._should_process_messages += int(
+            isinstance(
+                event,
+                (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
+            ),
+        )
+        self._hooks.setdefault(type(event), set()).add((hook, event))
+
+    def remove_hook(self, hook: Callable, event: Union[type, EventFilter]) -> None:
+        """Unregister hook from the given event filter."""
+        if isinstance(event, type):
+            event = event()
+        self._should_process_messages -= int(
+            isinstance(
+                event,
+                (NewMessage, MemberListChanged, GroupImageChanged, GroupNameChanged),
+            ),
+        )
+        self._hooks.get(type(event), set()).remove((hook, event))
+
+    async def is_configured(self) -> bool:
+        return await self.account.is_configured()
+
+    async def configure(self, email: str, password: str, **kwargs) -> None:
+        await self.account.set_config("addr", email)
+        await self.account.set_config("mail_pw", password)
+        for key, value in kwargs.items():
+            await self.account.set_config(key, value)
+        await self.account.configure()
+        self.logger.debug("Account configured")
+
+    async def run_forever(self) -> None:
+        """Process events forever."""
+        await self.run_until(lambda _: False)
+
+    async def run_until(self, func: Callable[[AttrDict], Union[bool, Coroutine]]) -> AttrDict:
+        """Process events until the given callable evaluates to True.
+
+        The callable should accept an AttrDict object representing the
+        last processed event. The event is returned when the callable
+        evaluates to True.
+        """
+        self.logger.debug("Listening to incoming events...")
+        if await self.is_configured():
+            await self.account.start_io()
+        await self._process_messages()  # Process old messages.
+        while True:
+            event = await self.account.wait_for_event()
+            event["type"] = EventType(event.type)
+            event["account"] = self.account
+            await self._on_event(event)
+            if event.type == EventType.INCOMING_MSG:
+                await self._process_messages()
+
+            stop = func(event)
+            if inspect.isawaitable(stop):
+                stop = await stop
+            if stop:
+                return event
+
+    async def _on_event(self, event: AttrDict, filter_type: Type[EventFilter] = RawEvent) -> None:
+        for hook, evfilter in self._hooks.get(filter_type, []):
+            if await evfilter.filter(event):
+                try:
+                    await hook(event)
+                except Exception as ex:
+                    self.logger.exception(ex)
+
+    async def _parse_command(self, event: AttrDict) -> None:
+        cmds = [hook[1].command for hook in self._hooks.get(NewMessage, []) if hook[1].command]
+        parts = event.message_snapshot.text.split(maxsplit=1)
+        payload = parts[1] if len(parts) > 1 else ""
+        cmd = parts.pop(0)
+
+        if "@" in cmd:
+            suffix = "@" + (await self.account.self_contact.get_snapshot()).address
+            if cmd.endswith(suffix):
+                cmd = cmd[: -len(suffix)]
+            else:
+                return
+
+        parts = cmd.split("_")
+        _payload = payload
+        while parts:
+            _cmd = "_".join(parts)
+            if _cmd in cmds:
+                break
+            _payload = (parts.pop() + " " + _payload).rstrip()
+
+        if parts:
+            cmd = _cmd
+            payload = _payload
+
+        event["command"], event["payload"] = cmd, payload
+
+    async def _on_new_msg(self, snapshot: AttrDict) -> None:
+        event = AttrDict(command="", payload="", message_snapshot=snapshot)
+        if not snapshot.is_info and snapshot.text.startswith(COMMAND_PREFIX):
+            await self._parse_command(event)
+        await self._on_event(event, NewMessage)
+
+    async def _handle_info_msg(self, snapshot: AttrDict) -> None:
+        event = AttrDict(message_snapshot=snapshot)
+
+        img_changed = parse_system_image_changed(snapshot.text)
+        if img_changed:
+            _, event["image_deleted"] = img_changed
+            await self._on_event(event, GroupImageChanged)
+            return
+
+        title_changed = parse_system_title_changed(snapshot.text)
+        if title_changed:
+            _, event["old_name"] = title_changed
+            await self._on_event(event, GroupNameChanged)
+            return
+
+        members_changed = parse_system_add_remove(snapshot.text)
+        if members_changed:
+            action, event["member"], _ = members_changed
+            event["member_added"] = action == "added"
+            await self._on_event(event, MemberListChanged)
+            return
+
+        self.logger.warning(
+            "ignoring unsupported system message id=%s text=%s",
+            snapshot.id,
+            snapshot.text,
+        )
+
+    async def _process_messages(self) -> None:
+        if self._should_process_messages:
+            for message in await self.account.get_fresh_messages_in_arrival_order():
+                snapshot = await message.get_snapshot()
+                await self._on_new_msg(snapshot)
+                if snapshot.is_info and snapshot.system_message_type != SystemMessageType.WEBXDC_INFO_MESSAGE:
+                    await self._handle_info_msg(snapshot)
+                await snapshot.message.mark_seen()
+
+
+class Bot(Client):
+    """Simple bot implementation that listent to events of a single account."""
+
+    async def configure(self, email: str, password: str, **kwargs) -> None:
+        kwargs.setdefault("bot", "1")
+        await super().configure(email, password, **kwargs)

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -0,0 +1,122 @@
+from enum import Enum, IntEnum
+
+COMMAND_PREFIX = "/"
+
+
+class ContactFlag(IntEnum):
+    VERIFIED_ONLY = 0x01
+    ADD_SELF = 0x02
+
+
+class ChatlistFlag(IntEnum):
+    ARCHIVED_ONLY = 0x01
+    NO_SPECIALS = 0x02
+    ADD_ALLDONE_HINT = 0x04
+    FOR_FORWARDING = 0x08
+
+
+class SpecialContactId(IntEnum):
+    SELF = 1
+    INFO = 2  # centered messages as "member added", used in all chats
+    DEVICE = 5  #  messages "update info" in the device-chat
+    LAST_SPECIAL = 9
+
+
+class EventType(str, Enum):
+    """Core event types"""
+
+    INFO = "Info"
+    SMTP_CONNECTED = "SmtpConnected"
+    IMAP_CONNECTED = "ImapConnected"
+    SMTP_MESSAGE_SENT = "SmtpMessageSent"
+    IMAP_MESSAGE_DELETED = "ImapMessageDeleted"
+    IMAP_MESSAGE_MOVED = "ImapMessageMoved"
+    NEW_BLOB_FILE = "NewBlobFile"
+    DELETED_BLOB_FILE = "DeletedBlobFile"
+    WARNING = "Warning"
+    ERROR = "Error"
+    ERROR_SELF_NOT_IN_GROUP = "ErrorSelfNotInGroup"
+    MSGS_CHANGED = "MsgsChanged"
+    REACTIONS_CHANGED = "ReactionsChanged"
+    INCOMING_MSG = "IncomingMsg"
+    INCOMING_MSG_BUNCH = "IncomingMsgBunch"
+    MSGS_NOTICED = "MsgsNoticed"
+    MSG_DELIVERED = "MsgDelivered"
+    MSG_FAILED = "MsgFailed"
+    MSG_READ = "MsgRead"
+    CHAT_MODIFIED = "ChatModified"
+    CHAT_EPHEMERAL_TIMER_MODIFIED = "ChatEphemeralTimerModified"
+    CONTACTS_CHANGED = "ContactsChanged"
+    LOCATION_CHANGED = "LocationChanged"
+    CONFIGURE_PROGRESS = "ConfigureProgress"
+    IMEX_PROGRESS = "ImexProgress"
+    IMEX_FILE_WRITTEN = "ImexFileWritten"
+    SECUREJOIN_INVITER_PROGRESS = "SecurejoinInviterProgress"
+    SECUREJOIN_JOINER_PROGRESS = "SecurejoinJoinerProgress"
+    CONNECTIVITY_CHANGED = "ConnectivityChanged"
+    SELFAVATAR_CHANGED = "SelfavatarChanged"
+    WEBXDC_STATUS_UPDATE = "WebxdcStatusUpdate"
+    WEBXDC_INSTANCE_DELETED = "WebxdcInstanceDeleted"
+
+
+class ChatType(IntEnum):
+    """Chat types"""
+
+    UNDEFINED = 0
+    SINGLE = 100
+    GROUP = 120
+    MAILINGLIST = 140
+    BROADCAST = 160
+
+
+class ChatVisibility(str, Enum):
+    """Chat visibility types"""
+
+    NORMAL = "Normal"
+    ARCHIVED = "Archived"
+    PINNED = "Pinned"
+
+
+class DownloadState(str, Enum):
+    """Message download state"""
+
+    DONE = "Done"
+    AVAILABLE = "Available"
+    FAILURE = "Failure"
+    IN_PROGRESS = "InProgress"
+
+
+class ViewType(str, Enum):
+    """Message view type."""
+
+    UNKNOWN = "Unknown"
+    TEXT = "Text"
+    IMAGE = "Image"
+    GIF = "Gif"
+    STICKER = "Sticker"
+    AUDIO = "Audio"
+    VOICE = "Voice"
+    VIDEO = "Video"
+    FILE = "File"
+    VIDEOCHAT_INVITATION = "VideochatInvitation"
+    WEBXDC = "Webxdc"
+
+
+class SystemMessageType(str, Enum):
+    """System message type."""
+
+    UNKNOWN = "Unknown"
+    GROUP_NAME_CHANGED = "GroupNameChanged"
+    GROUP_IMAGE_CHANGED = "GroupImageChanged"
+    MEMBER_ADDED_TO_GROUP = "MemberAddedToGroup"
+    MEMBER_REMOVED_FROM_GROUP = "MemberRemovedFromGroup"
+    AUTOCRYPT_SETUP_MESSAGE = "AutocryptSetupMessage"
+    SECUREJOIN_MESSAGE = "SecurejoinMessage"
+    LOCATION_STREAMING_ENABLED = "LocationStreamingEnabled"
+    LOCATION_ONLY = "LocationOnly"
+    CHAT_PROTECTION_ENABLED = "ChatProtectionEnabled"
+    CHAT_PROTECTION_DISABLED = "ChatProtectionDisabled"
+    WEBXDC_STATUS_UPDATE = "WebxdcStatusUpdate"
+    EPHEMERAL_TIMER_CHANGED = "EphemeralTimerChanged"
+    MULTI_DEVICE_SYNC = "MultiDeviceSync"
+    WEBXDC_INFO_MESSAGE = "WebxdcInfoMessage"

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from ._utils import AttrDict
+
+if TYPE_CHECKING:
+    from .account import Account
+    from .chat import Chat
+    from .rpc import Rpc
+
+
+@dataclass
+class Contact:
+    """
+    Contact API.
+
+    Essentially a wrapper for RPC, account ID and a contact ID.
+    """
+
+    account: "Account"
+    id: int
+
+    @property
+    def _rpc(self) -> "Rpc":
+        return self.account._rpc
+
+    async def block(self) -> None:
+        """Block contact."""
+        await self._rpc.block_contact(self.account.id, self.id)
+
+    async def unblock(self) -> None:
+        """Unblock contact."""
+        await self._rpc.unblock_contact(self.account.id, self.id)
+
+    async def delete(self) -> None:
+        """Delete contact."""
+        await self._rpc.delete_contact(self.account.id, self.id)
+
+    async def set_name(self, name: str) -> None:
+        """Change the name of this contact."""
+        await self._rpc.change_contact_name(self.account.id, self.id, name)
+
+    async def get_encryption_info(self) -> str:
+        """Get a multi-line encryption info, containing your fingerprint and
+        the fingerprint of the contact.
+        """
+        return await self._rpc.get_contact_encryption_info(self.account.id, self.id)
+
+    async def get_snapshot(self) -> AttrDict:
+        """Return a dictionary with a snapshot of all contact properties."""
+        snapshot = AttrDict(await self._rpc.get_contact(self.account.id, self.id))
+        snapshot["contact"] = self
+        return snapshot
+
+    async def create_chat(self) -> "Chat":
+        """Create or get an existing 1:1 chat for this contact."""
+        from .chat import Chat
+
+        return Chat(
+            self.account,
+            await self._rpc.create_chat_by_contact_id(self.account.id, self.id),
+        )

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -0,0 +1,49 @@
+from typing import TYPE_CHECKING, Dict, List
+
+from ._utils import AttrDict
+from .account import Account
+
+if TYPE_CHECKING:
+    from .rpc import Rpc
+
+
+class DeltaChat:
+    """
+    Delta Chat accounts manager.
+    This is the root of the object oriented API.
+    """
+
+    def __init__(self, rpc: "Rpc") -> None:
+        self.rpc = rpc
+
+    async def add_account(self) -> Account:
+        """Create a new account database."""
+        account_id = await self.rpc.add_account()
+        return Account(self, account_id)
+
+    async def get_all_accounts(self) -> List[Account]:
+        """Return a list of all available accounts."""
+        account_ids = await self.rpc.get_all_account_ids()
+        return [Account(self, account_id) for account_id in account_ids]
+
+    async def start_io(self) -> None:
+        """Start the I/O of all accounts."""
+        await self.rpc.start_io_for_all_accounts()
+
+    async def stop_io(self) -> None:
+        """Stop the I/O of all accounts."""
+        await self.rpc.stop_io_for_all_accounts()
+
+    async def maybe_network(self) -> None:
+        """Indicate that the network likely has come back or just that the network
+        conditions might have changed.
+        """
+        await self.rpc.maybe_network()
+
+    async def get_system_info(self) -> AttrDict:
+        """Get information about the Delta Chat core in this system."""
+        return AttrDict(await self.rpc.get_system_info())
+
+    async def set_translations(self, translations: Dict[str, str]) -> None:
+        """Set stock translation strings."""
+        await self.rpc.set_stock_strings(translations)

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -0,0 +1,286 @@
+"""High-level classes for event processing and filtering."""
+import inspect
+import re
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union
+
+from .const import EventType
+
+if TYPE_CHECKING:
+    from ._utils import AttrDict
+
+
+def _tuple_of(obj, type_: type) -> tuple:
+    if not obj:
+        return ()
+    if isinstance(obj, type_):
+        obj = (obj,)
+
+    if not all(isinstance(elem, type_) for elem in obj):
+        raise TypeError()
+    return tuple(obj)
+
+
+class EventFilter(ABC):
+    """The base event filter.
+
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, func: Optional[Callable] = None):
+        self.func = func
+
+    @abstractmethod
+    def __hash__(self) -> int:
+        """Object's unique hash"""
+
+    @abstractmethod
+    def __eq__(self, other) -> bool:
+        """Return True if two event filters are equal."""
+
+    def __ne__(self, other):
+        return not self == other
+
+    async def _call_func(self, event) -> bool:
+        if not self.func:
+            return True
+        res = self.func(event)
+        if inspect.isawaitable(res):
+            return await res
+        return res
+
+    @abstractmethod
+    async def filter(self, event):
+        """Return True-like value if the event passed the filter and should be
+        used, or False-like value otherwise.
+        """
+
+
+class RawEvent(EventFilter):
+    """Matches raw core events.
+
+    :param types: The types of event to match.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, types: Union[None, EventType, Iterable[EventType]] = None, **kwargs):
+        super().__init__(**kwargs)
+        try:
+            self.types = _tuple_of(types, EventType)
+        except TypeError as err:
+            raise TypeError(f"Invalid event type given: {types}") from err
+
+    def __hash__(self) -> int:
+        return hash((self.types, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, RawEvent):
+            return (self.types, self.func) == (other.types, other.func)
+        return False
+
+    async def filter(self, event: "AttrDict") -> bool:
+        if self.types and event.type not in self.types:
+            return False
+        return await self._call_func(event)
+
+
+class NewMessage(EventFilter):
+    """Matches whenever a new message arrives.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param pattern: if set, this Pattern will be used to filter the message by its text
+                    content.
+    :param command: If set, only match messages with the given command (ex. /help).
+                    Setting this property implies `is_info==False`.
+    :param is_bot: If set to True only match messages sent by bots, if set to None
+                   match messages from bots and users. If omitted or set to False
+                   only messages from users will be matched.
+    :param is_info: If set to True only match info/system messages, if set to False
+                    only match messages that are not info/system messages. If omitted
+                    info/system messages as well as normal messages will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(
+        self,
+        pattern: Union[
+            None,
+            str,
+            Callable[[str], bool],
+            re.Pattern,
+        ] = None,
+        command: Optional[str] = None,
+        is_bot: Optional[bool] = False,
+        is_info: Optional[bool] = None,
+        func: Optional[Callable[["AttrDict"], bool]] = None,
+    ) -> None:
+        super().__init__(func=func)
+        self.is_bot = is_bot
+        self.is_info = is_info
+        if command is not None and not isinstance(command, str):
+            raise TypeError("Invalid command")
+        self.command = command
+        if self.is_info and self.command:
+            raise AttributeError("Can not use command and is_info at the same time.")
+        if isinstance(pattern, str):
+            pattern = re.compile(pattern)
+        if isinstance(pattern, re.Pattern):
+            self.pattern: Optional[Callable] = pattern.match
+        elif not pattern or callable(pattern):
+            self.pattern = pattern
+        else:
+            raise TypeError("Invalid pattern type")
+
+    def __hash__(self) -> int:
+        return hash((self.pattern, self.command, self.is_bot, self.is_info, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, NewMessage):
+            return (
+                self.pattern,
+                self.command,
+                self.is_bot,
+                self.is_info,
+                self.func,
+            ) == (
+                other.pattern,
+                other.command,
+                other.is_bot,
+                other.is_info,
+                other.func,
+            )
+        return False
+
+    async def filter(self, event: "AttrDict") -> bool:
+        if self.is_bot is not None and self.is_bot != event.message_snapshot.is_bot:
+            return False
+        if self.is_info is not None and self.is_info != event.message_snapshot.is_info:
+            return False
+        if self.command and self.command != event.command:
+            return False
+        if self.pattern:
+            match = self.pattern(event.message_snapshot.text)
+            if inspect.isawaitable(match):
+                match = await match
+            if not match:
+                return False
+        return await super()._call_func(event)
+
+
+class MemberListChanged(EventFilter):
+    """Matches when a group member is added or removed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param added: If set to True only match if a member was added, if set to False
+                  only match if a member was removed. If omitted both, member additions
+                  and removals, will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, added: Optional[bool] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.added = added
+
+    def __hash__(self) -> int:
+        return hash((self.added, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, MemberListChanged):
+            return (self.added, self.func) == (other.added, other.func)
+        return False
+
+    async def filter(self, event: "AttrDict") -> bool:
+        if self.added is not None and self.added != event.member_added:
+            return False
+        return await self._call_func(event)
+
+
+class GroupImageChanged(EventFilter):
+    """Matches when the group image is changed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param deleted: If set to True only match if the image was deleted, if set to False
+                    only match if a new image was set. If omitted both, image changes and
+                    removals, will be matched.
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __init__(self, deleted: Optional[bool] = None, **kwargs):
+        super().__init__(**kwargs)
+        self.deleted = deleted
+
+    def __hash__(self) -> int:
+        return hash((self.deleted, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, GroupImageChanged):
+            return (self.deleted, self.func) == (other.deleted, other.func)
+        return False
+
+    async def filter(self, event: "AttrDict") -> bool:
+        if self.deleted is not None and self.deleted != event.image_deleted:
+            return False
+        return await self._call_func(event)
+
+
+class GroupNameChanged(EventFilter):
+    """Matches when the group name is changed.
+
+    Warning: registering a handler for this event will cause the messages
+    to be marked as read. Its usage is mainly intended for bots.
+
+    :param func: A Callable (async or not) function that should accept the event as input
+                 parameter, and return a bool value indicating whether the event
+                 should be dispatched or not.
+    """
+
+    def __hash__(self) -> int:
+        return hash((GroupNameChanged, self.func))
+
+    def __eq__(self, other) -> bool:
+        if isinstance(other, GroupNameChanged):
+            return self.func == other.func
+        return False
+
+    async def filter(self, event: "AttrDict") -> bool:
+        return await self._call_func(event)
+
+
+class HookCollection:
+    """
+    Helper class to collect event hooks that can later be added to a Delta Chat client.
+    """
+
+    def __init__(self) -> None:
+        self._hooks: Set[Tuple[Callable, Union[type, EventFilter]]] = set()
+
+    def __iter__(self) -> Iterator[Tuple[Callable, Union[type, EventFilter]]]:
+        return iter(self._hooks)
+
+    def on(self, event: Union[type, EventFilter]) -> Callable:  # noqa
+        """Register decorated function as listener for the given event."""
+        if isinstance(event, type):
+            event = event()
+        assert isinstance(event, EventFilter), "Invalid event filter"
+
+        def _decorator(func) -> Callable:
+            self._hooks.add((func, event))
+            return func
+
+        return _decorator

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -0,0 +1,52 @@
+import json
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Union
+
+from ._utils import AttrDict
+from .contact import Contact
+
+if TYPE_CHECKING:
+    from .account import Account
+    from .rpc import Rpc
+
+
+@dataclass
+class Message:
+    """Delta Chat Message object."""
+
+    account: "Account"
+    id: int
+
+    @property
+    def _rpc(self) -> "Rpc":
+        return self.account._rpc
+
+    async def send_reaction(self, *reaction: str):
+        """Send a reaction to this message."""
+        await self._rpc.send_reaction(self.account.id, self.id, reaction)
+
+    async def get_snapshot(self) -> AttrDict:
+        """Get a snapshot with the properties of this message."""
+        from .chat import Chat
+
+        snapshot = AttrDict(await self._rpc.get_message(self.account.id, self.id))
+        snapshot["chat"] = Chat(self.account, snapshot.chat_id)
+        snapshot["sender"] = Contact(self.account, snapshot.from_id)
+        snapshot["message"] = self
+        return snapshot
+
+    async def mark_seen(self) -> None:
+        """Mark the message as seen."""
+        await self._rpc.markseen_msgs(self.account.id, [self.id])
+
+    async def send_webxdc_status_update(self, update: Union[dict, str], description: str) -> None:
+        """Send a webxdc status update. This message must be a webxdc."""
+        if not isinstance(update, str):
+            update = json.dumps(update)
+        await self._rpc.send_webxdc_status_update(self.account.id, self.id, update, description)
+
+    async def get_webxdc_status_updates(self, last_known_serial: int = 0) -> list:
+        return json.loads(await self._rpc.get_webxdc_status_updates(self.account.id, self.id, last_known_serial))
+
+    async def get_webxdc_info(self) -> dict:
+        return await self._rpc.get_webxdc_info(self.account.id, self.id)

deltachat-rpc-client/src/deltachat_rpc_client/py.typed

@@ -0,0 +1 @@
+# PEP 561 marker file. See https://peps.python.org/pep-0561/

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -0,0 +1,109 @@
+import asyncio
+import json
+import os
+from typing import AsyncGenerator, List, Optional
+
+import aiohttp
+import pytest_asyncio
+
+from . import Account, AttrDict, Bot, Client, DeltaChat, EventType, Message
+from .rpc import Rpc
+
+
+async def get_temp_credentials() -> dict:
+    url = os.getenv("DCC_NEW_TMP_EMAIL")
+    assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
+
+    # Replace default 5 minute timeout with a 1 minute timeout.
+    timeout = aiohttp.ClientTimeout(total=60)
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url, timeout=timeout) as response:
+            return json.loads(await response.text())
+
+
+class ACFactory:
+    def __init__(self, deltachat: DeltaChat) -> None:
+        self.deltachat = deltachat
+
+    async def get_unconfigured_account(self) -> Account:
+        return await self.deltachat.add_account()
+
+    async def get_unconfigured_bot(self) -> Bot:
+        return Bot(await self.get_unconfigured_account())
+
+    async def new_preconfigured_account(self) -> Account:
+        """Make a new account with configuration options set, but configuration not started."""
+        credentials = await get_temp_credentials()
+        account = await self.get_unconfigured_account()
+        await account.set_config("addr", credentials["email"])
+        await account.set_config("mail_pw", credentials["password"])
+        assert not await account.is_configured()
+        return account
+
+    async def new_configured_account(self) -> Account:
+        account = await self.new_preconfigured_account()
+        await account.configure()
+        assert await account.is_configured()
+        return account
+
+    async def new_configured_bot(self) -> Bot:
+        credentials = await get_temp_credentials()
+        bot = await self.get_unconfigured_bot()
+        await bot.configure(credentials["email"], credentials["password"])
+        return bot
+
+    async def get_online_account(self) -> Account:
+        account = await self.new_configured_account()
+        await account.start_io()
+        return account
+
+    async def get_online_accounts(self, num: int) -> List[Account]:
+        return await asyncio.gather(*[self.get_online_account() for _ in range(num)])
+
+    async def send_message(
+        self,
+        to_account: Account,
+        from_account: Optional[Account] = None,
+        text: Optional[str] = None,
+        file: Optional[str] = None,
+        group: Optional[str] = None,
+    ) -> Message:
+        if not from_account:
+            from_account = (await self.get_online_accounts(1))[0]
+        to_contact = await from_account.create_contact(await to_account.get_config("addr"))
+        if group:
+            to_chat = await from_account.create_group(group)
+            await to_chat.add_contact(to_contact)
+        else:
+            to_chat = await to_contact.create_chat()
+        return await to_chat.send_message(text=text, file=file)
+
+    async def process_message(
+        self,
+        to_client: Client,
+        from_account: Optional[Account] = None,
+        text: Optional[str] = None,
+        file: Optional[str] = None,
+        group: Optional[str] = None,
+    ) -> AttrDict:
+        await self.send_message(
+            to_account=to_client.account,
+            from_account=from_account,
+            text=text,
+            file=file,
+            group=group,
+        )
+
+        return await to_client.run_until(lambda e: e.type == EventType.INCOMING_MSG)
+
+
+@pytest_asyncio.fixture
+async def rpc(tmp_path) -> AsyncGenerator:
+    rpc_server = Rpc(accounts_dir=str(tmp_path / "accounts"))
+    async with rpc_server:
+        yield rpc_server
+
+
+@pytest_asyncio.fixture
+async def acfactory(rpc) -> AsyncGenerator:
+    yield ACFactory(DeltaChat(rpc))