chore(release): prepare for 1.142.6

If user has not set any settings manually
and provider is not configured,
default to strict TLS checks.

Bug was introduced in
<https://github.com/deltachat/deltachat-core-rust/pull/5854>
(commit 6b4532a08e)
and affects released core 1.142.4 and 1.142.5.

The problem only affects accounts configured
using these core versions with provider
not in the provider database or when using advanced settings.

.github/dependabot.yml

@@ -7,3 +7,10 @@ updates:
     commit-message:
       prefix: "chore(cargo)"
     open-pull-requests-limit: 50
+
+  # Keep GitHub Actions up to date.
+  # <https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot>
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -24,9 +24,11 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.74.0
+      RUSTUP_TOOLCHAIN: 1.80.1
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Install rustfmt and clippy
         run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
       - name: Cache rust cargo artifacts
@@ -38,12 +40,26 @@ jobs:
       - name: Check
         run: cargo check --workspace --all-targets --all-features
 
+  npm_constants:
+    name: Check if node constants are up to date
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Rebuild constants
+        run: npm run build:core:constants
+      - name: Check that constants are not changed
+        run: git diff --exit-code
+
   cargo_deny:
     name: cargo deny
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: EmbarkStudios/cargo-deny-action@v2
         with:
           arguments: --all-features --workspace
           command: check
@@ -53,7 +69,9 @@ jobs:
     name: Check provider database
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Check provider database
         run: scripts/update-provider-database.sh
 
@@ -63,8 +81,9 @@ jobs:
     env:
       RUSTDOCFLAGS: -Dwarnings
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
       - name: Rustdoc
@@ -76,18 +95,20 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.74.0
+            rust: 1.80.1
           - os: windows-latest
-            rust: 1.74.0
+            rust: 1.80.1
           - os: macos-latest
-            rust: 1.74.0
+            rust: 1.80.1
 
-          # Minimum Supported Rust Version = 1.70.0
+          # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest
-            rust: 1.70.0
+            rust: 1.77.0
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install Rust ${{ matrix.rust }}
         run: rustup toolchain install --profile minimal ${{ matrix.rust }}
@@ -96,10 +117,20 @@ jobs:
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
 
+      - name: Install nextest
+        uses: taiki-e/install-action@v2
+        with:
+          tool: nextest
+
       - name: Tests
         env:
           RUST_BACKTRACE: 1
-        run: cargo test --workspace
+        run: cargo nextest run --workspace
+
+      - name: Doc-Tests
+        env:
+          RUST_BACKTRACE: 1
+        run: cargo test --workspace --doc
 
       - name: Test cargo vendor
         run: cargo vendor
@@ -111,7 +142,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -120,7 +153,7 @@ jobs:
         run: cargo build -p deltachat_ffi --features jsonrpc
 
       - name: Upload C library
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug/libdeltachat.a
@@ -133,7 +166,9 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -142,7 +177,7 @@ jobs:
         run: cargo build -p deltachat-rpc-server
 
       - name: Upload deltachat-rpc-server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
@@ -152,8 +187,9 @@ jobs:
     name: Python lint
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install tox
         run: pip install tox
@@ -193,16 +229,18 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Download libdeltachat.a
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -243,10 +281,12 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -254,7 +294,7 @@ jobs:
         run: pip install tox
 
       - name: Download deltachat-rpc-server
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: target/debug

.github/workflows/deltachat-rpc-server.yml

@@ -21,69 +21,65 @@ jobs:
   # Build a version statically linked against musl libc
   # to avoid problems with glibc version incompatibility.
   build_linux:
-    name: Cross-compile deltachat-rpc-server for x86_64, i686, aarch64 and armv7 Linux
-    runs-on: ubuntu-22.04
+    name: Linux
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [aarch64, armv7l, armv6l, i686, x86_64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install ziglang
-        run: pip install wheel ziglang==0.11.0
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
-        run: sh scripts/zig-rpc-server.sh
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-linux
 
-      - name: Upload dist directory with Linux binaries
-        uses: actions/upload-artifact@v3
+      - name: Upload binary
+        uses: actions/upload-artifact@v4
         with:
-          name: linux
-          path: dist/
+          name: deltachat-rpc-server-${{ matrix.arch }}-linux
+          path: result/bin/deltachat-rpc-server
           if-no-files-found: error
 
   build_windows:
-    name: Build deltachat-rpc-server for Windows
+    name: Windows
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: windows-latest
-            artifact: win32.exe
-            path: deltachat-rpc-server.exe
-            target: i686-pc-windows-msvc
-
-          - os: windows-latest
-            artifact: win64.exe
-            path: deltachat-rpc-server.exe
-            target: x86_64-pc-windows-msvc
-
-    runs-on: ${{ matrix.os }}
+        arch: [win32, win64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
-      - name: Setup rust target
-        run: rustup target add ${{ matrix.target }}
-
-      - name: Build
-        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
+      - name: Build deltachat-rpc-server binaries
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: deltachat-rpc-server-${{ matrix.artifact }}
-          path: target/${{ matrix.target}}/release/${{ matrix.path }}
+          name: deltachat-rpc-server-${{ matrix.arch }}
+          path: result/bin/deltachat-rpc-server.exe
           if-no-files-found: error
 
   build_macos:
-    name: Build deltachat-rpc-server for macOS
+    name: macOS
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch: x86_64
-          - arch: aarch64
+        arch: [x86_64, aarch64]
 
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Setup rust target
         run: rustup target add ${{ matrix.arch }}-apple-darwin
@@ -92,61 +88,140 @@ jobs:
         run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.arch }}-apple-darwin --features vendored
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-macos
           path: target/${{ matrix.arch }}-apple-darwin/release/deltachat-rpc-server
           if-no-files-found: error
 
+  build_android:
+    name: Android
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [arm64-v8a, armeabi-v7a]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+
+      - name: Build deltachat-rpc-server binaries
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-android
+
+      - name: Upload binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: deltachat-rpc-server-${{ matrix.arch }}-android
+          path: result/bin/deltachat-rpc-server
+          if-no-files-found: error
+
   publish:
     name: Build wheels and upload binaries to the release
     needs: ["build_linux", "build_windows", "build_macos"]
+    environment:
+      name: pypi
+      url: https://pypi.org/p/deltachat-rpc-server
     permissions:
+      id-token: write
       contents: write
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Download Linux binaries
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
         with:
-          name: linux
-          path: dist/
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
-      - name: Download win32 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux aarch64 binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win32.exe
-          path: deltachat-rpc-server-win32.exe.d
+          name: deltachat-rpc-server-aarch64-linux
+          path: deltachat-rpc-server-aarch64-linux.d
 
-      - name: Download win64 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux armv7l binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win64.exe
-          path: deltachat-rpc-server-win64.exe.d
+          name: deltachat-rpc-server-armv7l-linux
+          path: deltachat-rpc-server-armv7l-linux.d
+
+      - name: Download Linux armv6l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv6l-linux
+          path: deltachat-rpc-server-armv6l-linux.d
+
+      - name: Download Linux i686 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-i686-linux
+          path: deltachat-rpc-server-i686-linux.d
+
+      - name: Download Linux x86_64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-linux
+          path: deltachat-rpc-server-x86_64-linux.d
+
+      - name: Download Win32 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win32
+          path: deltachat-rpc-server-win32.d
+
+      - name: Download Win64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win64
+          path: deltachat-rpc-server-win64.d
 
       - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-x86_64-macos
           path: deltachat-rpc-server-x86_64-macos.d
 
       - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-aarch64-macos
           path: deltachat-rpc-server-aarch64-macos.d
 
-      - name: Flatten dist/ directory
+      - name: Download Android binary for arm64-v8a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-arm64-v8a-android
+          path: deltachat-rpc-server-arm64-v8a-android.d
+
+      - name: Download Android binary for armeabi-v7a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armeabi-v7a-android
+          path: deltachat-rpc-server-armeabi-v7a-android.d
+
+      - name: Create bin/ directory
         run: |
-          mv deltachat-rpc-server-win32.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
-          mv deltachat-rpc-server-win64.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
-          mv deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-macos
-          mv deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-macos
+          mkdir -p bin
+          mv deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server bin/deltachat-rpc-server-aarch64-linux
+          mv deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server bin/deltachat-rpc-server-armv7l-linux
+          mv deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server bin/deltachat-rpc-server-armv6l-linux
+          mv deltachat-rpc-server-i686-linux.d/deltachat-rpc-server bin/deltachat-rpc-server-i686-linux
+          mv deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server bin/deltachat-rpc-server-x86_64-linux
+          mv deltachat-rpc-server-win32.d/deltachat-rpc-server.exe bin/deltachat-rpc-server-win32.exe
+          mv deltachat-rpc-server-win64.d/deltachat-rpc-server.exe bin/deltachat-rpc-server-win64.exe
+          mv deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server bin/deltachat-rpc-server-x86_64-macos
+          mv deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server bin/deltachat-rpc-server-aarch64-macos
+          mv deltachat-rpc-server-arm64-v8a-android.d/deltachat-rpc-server bin/deltachat-rpc-server-arm64-v8a-android
+          mv deltachat-rpc-server-armeabi-v7a-android.d/deltachat-rpc-server bin/deltachat-rpc-server-armeabi-v7a-android
+
+      - name: List binaries
+        run: ls -l bin/
 
       # Python 3.11 is needed for tomllib used in scripts/wheel-rpc-server.py
       - name: Install python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
@@ -154,15 +229,178 @@ jobs:
         run: pip install wheel
 
       - name: Build deltachat-rpc-server Python wheels and source package
-        run: scripts/wheel-rpc-server.py
+        run: |
+          mkdir -p dist
+          nix build .#deltachat-rpc-server-x86_64-linux-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-armv7l-linux-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-armv6l-linux-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-aarch64-linux-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-i686-linux-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-win64-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-win32-wheel
+          cp result/*.whl dist/
+          nix build .#deltachat-rpc-server-source
+          cp result/*.tar.gz dist/
+          python3 scripts/wheel-rpc-server.py x86_64-darwin bin/deltachat-rpc-server-x86_64-macos
+          python3 scripts/wheel-rpc-server.py aarch64-darwin bin/deltachat-rpc-server-aarch64-macos
+          mv *.whl dist/
 
-      - name: List downloaded artifacts
+      - name: List artifacts
         run: ls -l dist/
 
       - name: Upload binaries to the GitHub release
+        if: github.event_name == 'release'
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |
           gh release upload ${{ github.ref_name }} \
             --repo ${{ github.repository }} \
-            dist/*
+            bin/* dist/*
+
+      - name: Publish deltachat-rpc-client to PyPI
+        if: github.event_name == 'release'
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish_npm_package:
+    name: Build & Publish npm prebuilds and deltachat-rpc-server
+    needs: ["build_linux", "build_windows", "build_macos"]
+    runs-on: "ubuntu-latest"
+    permissions:
+      id-token: write
+
+      # Needed to publish the binaries to the release.
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Download Linux aarch64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-aarch64-linux
+          path: deltachat-rpc-server-aarch64-linux.d
+
+      - name: Download Linux armv7l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv7l-linux
+          path: deltachat-rpc-server-armv7l-linux.d
+
+      - name: Download Linux armv6l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv6l-linux
+          path: deltachat-rpc-server-armv6l-linux.d
+
+      - name: Download Linux i686 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-i686-linux
+          path: deltachat-rpc-server-i686-linux.d
+
+      - name: Download Linux x86_64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-linux
+          path: deltachat-rpc-server-x86_64-linux.d
+
+      - name: Download Win32 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win32
+          path: deltachat-rpc-server-win32.d
+
+      - name: Download Win64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win64
+          path: deltachat-rpc-server-win64.d
+
+      - name: Download macOS binary for x86_64
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-macos
+          path: deltachat-rpc-server-x86_64-macos.d
+
+      - name: Download macOS binary for aarch64
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-aarch64-macos
+          path: deltachat-rpc-server-aarch64-macos.d
+
+      - name: Download Android binary for arm64-v8a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-arm64-v8a-android
+          path: deltachat-rpc-server-arm64-v8a-android.d
+
+      - name: Download Android binary for armeabi-v7a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armeabi-v7a-android
+          path: deltachat-rpc-server-armeabi-v7a-android.d
+
+      - name: make npm packets for prebuilds and `@deltachat/stdio-rpc-server`
+        run: |
+          cd deltachat-rpc-server/npm-package
+
+          python --version
+
+          python scripts/pack_binary_for_platform.py aarch64-unknown-linux-musl     ../../deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py armv7-unknown-linux-musleabihf ../../deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py arm-unknown-linux-musleabihf   ../../deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py i686-unknown-linux-musl        ../../deltachat-rpc-server-i686-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py x86_64-unknown-linux-musl      ../../deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py i686-pc-windows-gnu            ../../deltachat-rpc-server-win32.d/deltachat-rpc-server.exe 
+          python scripts/pack_binary_for_platform.py x86_64-pc-windows-gnu          ../../deltachat-rpc-server-win64.d/deltachat-rpc-server.exe 
+          python scripts/pack_binary_for_platform.py x86_64-apple-darwin            ../../deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py aarch64-apple-darwin           ../../deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py aarch64-linux-android          ../../deltachat-rpc-server-arm64-v8a-android.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py armv7-linux-androideabi        ../../deltachat-rpc-server-armeabi-v7a-android.d/deltachat-rpc-server
+
+          ls -lah platform_package
+
+          for platform in ./platform_package/*; do npm pack "$platform"; done
+          npm pack    
+          ls -lah
+
+      - name: Upload to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deltachat-rpc-server-npm-package
+          path: deltachat-rpc-server/npm-package/*.tgz
+          if-no-files-found: error
+
+      - name: Upload npm packets to the GitHub release
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh release upload ${{ github.ref_name }} \
+            --repo ${{ github.repository }} \
+            deltachat-rpc-server/npm-package/*.tgz
+
+      # Configure Node.js for publishing.
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Publish npm packets for prebuilds and `@deltachat/stdio-rpc-server`
+        if: github.event_name == 'release'
+        working-directory: deltachat-rpc-server/npm-package
+        run: |
+          ls -lah platform_package
+          for platform in *.tgz; do npm publish --provenance "$platform" --access public; done
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/dependabot.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1.1.1
+        uses: dependabot/fetch-metadata@v2.2.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve a PR

.github/workflows/jsonrpc-client-npm-package.yml

@@ -1,81 +1,38 @@
-name: "jsonrpc js client build"
+name: "Publish @deltachat/jsonrpc-client"
 on:
-  pull_request:
-  push:
-    tags:
-      - "*"
-      - "!py-*"
+  workflow_dispatch:
+  release:
+    types: [published]
 
 jobs:
   pack-module:
-    name: "Package @deltachat/jsonrpc-client and upload to download.delta.chat"
+    name: "Publish @deltachat/jsonrpc-client"
     runs-on: ubuntu-20.04
+    permissions:
+      id-token: write
+      contents: read
     steps:
-      - name: Install tree
-        run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
         with:
-          node-version: "18"
-      - name: Get tag
-        id: tag
-        uses: dawidd6/action-get-tag@v1
-        continue-on-error: true
-      - name: Get Pull Request ID
-        id: prepare
-        run: |
-          tag=${{ steps.tag.outputs.tag }}
-          if [ -z "$tag" ]; then
-            node -e "console.log('DELTACHAT_JSONRPC_TAR_GZ=deltachat-jsonrpc-client-' + '${{ github.ref }}'.split('/')[2] + '.tar.gz')" >> $GITHUB_ENV
-          else
-            echo "DELTACHAT_JSONRPC_TAR_GZ=deltachat-jsonrpc-client-${{ steps.tag.outputs.tag }}.tar.gz" >> $GITHUB_ENV
-            echo "No preview will be uploaded this time, but the $tag release"
-          fi
-      - name: System info
-        run: |
-          npm --version
-          node --version
-          echo $DELTACHAT_JSONRPC_TAR_GZ
+          show-progress: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          registry-url: "https://registry.npmjs.org"
+
       - name: Install dependencies without running scripts
         working-directory: deltachat-jsonrpc/typescript
         run: npm install --ignore-scripts
+
       - name: Package
-        shell: bash
         working-directory: deltachat-jsonrpc/typescript
         run: |
           npm run build
           npm pack .
-          ls -lah
-          mv $(find deltachat-jsonrpc-client-*) $DELTACHAT_JSONRPC_TAR_GZ
-      - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
-        with:
-          name: deltachat-jsonrpc-client.tgz
-          path: deltachat-jsonrpc/typescript/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
-      # Upload to download.delta.chat/node/preview/
-      - name: Upload deltachat-jsonrpc-client preview to download.delta.chat/node/preview/
-        if: ${{ ! steps.tag.outputs.tag }}
-        id: upload-preview
-        shell: bash
-        run: |
-          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
-          chmod 600 __TEMP_INPUT_KEY_FILE
-          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
-        continue-on-error: true
-      - name: Post links to details
-        if: steps.upload-preview.outcome == 'success'
-        run: node ./node/scripts/postLinksToDetails.js
+
+      - name: Publish
+        working-directory: deltachat-jsonrpc/typescript
+        run: npm publish --provenance deltachat-jsonrpc-client-* --access public
         env:
-          URL: preview/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MSG_CONTEXT: Download the deltachat-jsonrpc-client.tgz
-      # Upload to download.delta.chat/node/
-      - name: Upload deltachat-jsonrpc-client build to download.delta.chat/node/
-        if: ${{ steps.tag.outputs.tag }}
-        id: upload
-        shell: bash
-        run: |
-          echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
-          chmod 600 __TEMP_INPUT_KEY_FILE
-          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-jsonrpc/typescript/$DELTACHAT_JSONRPC_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/jsonrpc.yml

@@ -14,9 +14,11 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Use Node.js 18.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18.x
       - name: Add Rust cache

.github/workflows/node-docs.yml

@@ -14,10 +14,12 @@ jobs:
   generate:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Use Node.js 18.x
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18.x
 
@@ -29,7 +31,7 @@ jobs:
           mv docs js
 
       - name: Upload
-        uses: horochx/deploy-via-scp@v1.0.1
+        uses: horochx/deploy-via-scp@1.1.0
         with:
           user: ${{ secrets.USERNAME }}
           key: ${{ secrets.KEY }}

.github/workflows/node-package.yml

@@ -14,9 +14,10 @@ jobs:
       matrix:
         os: [macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-node@v4
         with:
           node-version: "18"
       - name: System info
@@ -28,7 +29,7 @@ jobs:
           node --version
 
       - name: Cache node modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ env.APPDATA }}/npm-cache
@@ -36,7 +37,7 @@ jobs:
           key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/
@@ -56,7 +57,7 @@ jobs:
           tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: node/${{ matrix.os }}.tar.gz
@@ -74,9 +75,10 @@ jobs:
       - name: Change working directory owner
         run: chown root:root .
 
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-node@v4
         with:
           node-version: "18"
       - run: apt-get update
@@ -97,7 +99,7 @@ jobs:
           node --version
 
       - name: Cache node modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ env.APPDATA }}/npm-cache
@@ -105,7 +107,7 @@ jobs:
           key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/
@@ -125,7 +127,7 @@ jobs:
           tar -zcvf "linux.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux
           path: node/linux.tar.gz
@@ -137,9 +139,10 @@ jobs:
     steps:
       - name: Install tree
         run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v2
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-node@v4
         with:
           node-version: "18"
       - name: Get tag
@@ -165,25 +168,25 @@ jobs:
           node --version
           echo $DELTACHAT_NODE_TAR_GZ
       - name: Download Linux prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: linux
       - name: Download macOS prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: macos-latest
       - name: Download Windows prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: windows-latest
       - shell: bash
         run: |
           mkdir node/prebuilds
-          tar -xvzf linux/linux.tar.gz -C node/prebuilds
-          tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
-          tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
+          tar -xvzf linux.tar.gz -C node/prebuilds
+          tar -xvzf macos-latest.tar.gz -C node/prebuilds
+          tar -xvzf windows-latest.tar.gz -C node/prebuilds
           tree node/prebuilds
-          rm -rf linux macos-latest windows-latest
+          rm -f linux.tar.gz macos-latest.tar.gz windows-latest.tar.gz
       - name: Install dependencies without running scripts
         run: |
           npm install --ignore-scripts
@@ -201,7 +204,7 @@ jobs:
           ls -lah
           mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
       - name: Upload prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-node.tgz
           path: ${{ env.DELTACHAT_NODE_TAR_GZ }}

.github/workflows/node-tests.yml

@@ -23,9 +23,10 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-node@v4
         with:
           node-version: "18"
       - name: System info
@@ -37,7 +38,7 @@ jobs:
           node --version
 
       - name: Cache node modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ${{ env.APPDATA }}/npm-cache
@@ -45,7 +46,7 @@ jobs:
           key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
 
       - name: Cache cargo index
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/

.github/workflows/publish-deltachat-rpc-client-pypi.yml

@@ -0,0 +1,47 @@
+name: Publish deltachat-rpc-client to PyPI
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - name: Install pypa/build
+        run: python3 -m pip install build
+      - name: Build a binary wheel and a source tarball
+        working-directory: deltachat-rpc-client
+        run: python3 -m build
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: deltachat-rpc-client/dist/
+
+  publish-to-pypi:
+    name: Publish Python distribution to PyPI
+    if: github.event_name == 'release'
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/deltachat-rpc-client
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish deltachat-rpc-client to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/repl.yml

@@ -10,15 +10,17 @@ on:
 jobs:
   build_repl:
     name: Build REPL example
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build
-        run: cargo build -p deltachat-repl --features vendored
-
+        run: nix build .#deltachat-repl-win64
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: repl.exe
-          path: "target/debug/deltachat-repl.exe"
+          path: "result/bin/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -1,24 +1,91 @@
-name: Build & Deploy Documentation on rs.delta.chat
+name: Build & deploy documentation on rs.delta.chat, c.delta.chat, and py.delta.chat
 
 on:
   push:
     branches:
       - main
+      - build_jsonrpc_docs_ci
 
 jobs:
-  build:
+  build-rs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat --no-deps --document-private-items
       - name: Upload to rs.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/rs/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
+
+  build-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build Python documentation
+        run: nix build .#python-docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+
+  build-c:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build C documentation
+        run: nix build .#docs
+      - name: Upload to c.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
+
+  build-ts:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deltachat-jsonrpc/typescript
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+      - name: npm install
+        run: npm install
+      - name: npm run build
+        run: npm run build
+      - name: Run docs script
+        run: npm run docs
+      - name: Upload to js.jsonrpc.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"

.github/workflows/upload-ffi-docs.yml

@@ -1,5 +1,5 @@
 # GitHub Actions workflow
-# to build `deltachat_fii` crate documentation
+# to build `deltachat_ffi` crate documentation
 # and upload it to <https://cffi.delta.chat/>
 
 name: Build & Deploy Documentation on cffi.delta.chat
@@ -14,15 +14,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat_ffi --no-deps
       - name: Upload to cffi.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/cffi/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

.github/workflows/upload-python-docs.yml

@@ -1,25 +0,0 @@
-name: Build & Deploy Documentation on py.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Build Python documentation
-        run: scripts/build-python-docs.sh
-      - name: Upload to py.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: delta
-          KEY: ${{ secrets.CODESPEAK_KEY }}
-          HOST: "lists.codespeak.net"
-          SOURCE: "dist/html/"
-          TARGET: "/home/delta/build/master"

.gitignore

@@ -33,7 +33,7 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
-.vscode/launch.json
+.vscode
 python/accounts.txt
 python/all-testaccounts.txt
 tmp/
@@ -44,3 +44,10 @@ node/build/
 node/dist/
 node/prebuilds/
 node/.nyc_output/
+
+# Nix symlink.
+result
+
+# direnv
+.envrc
+.direnv

CMakeLists.txt

@@ -12,26 +12,22 @@ else()
        set(DYNAMIC_EXT "dll")
 endif()
 
+if(DEFINED ENV{CARGO_BUILD_TARGET})
+    set(ARCH_DIR "$ENV{CARGO_BUILD_TARGET}")
+else()
+    set(ARCH_DIR "./")
+endif()
+
 add_custom_command(
 	OUTPUT
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
-	"target/release/pkgconfig/deltachat.pc"
+	"${CMAKE_BINARY_DIR}/target/release/libdeltachat.a"
+	"${CMAKE_BINARY_DIR}/target/release/libdeltachat.${DYNAMIC_EXT}"
+	"${CMAKE_BINARY_DIR}/target/release/pkgconfig/deltachat.pc"
         COMMAND
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features --features jsonrpc
-
-	# Build in `deltachat-ffi` directory instead of using
-	# `--package deltachat_ffi` to avoid feature resolver version
-	# "1" bug which makes `--no-default-features` affect only
-	# `deltachat`, but not `deltachat-ffi` package.
-	#
-	# We can't enable version "2" resolver [1] because it is not
-	# stable yet on rust 1.50.0.
-	#
-	# [1] https://doc.rust-lang.org/nightly/cargo/reference/features.html#resolver-version-2-command-line-flags
+	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --no-default-features --features jsonrpc
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 
@@ -39,12 +35,12 @@ add_custom_target(
 	lib_deltachat
 	ALL
 	DEPENDS
-	"target/release/libdeltachat.a"
-	"target/release/libdeltachat.${DYNAMIC_EXT}"
-	"target/release/pkgconfig/deltachat.pc"
+	"${CMAKE_BINARY_DIR}/target/release/libdeltachat.a"
+	"${CMAKE_BINARY_DIR}/target/release/libdeltachat.${DYNAMIC_EXT}"
+	"${CMAKE_BINARY_DIR}/target/release/pkgconfig/deltachat.pc"
 )
 
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
-install(FILES "target/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+install(FILES "${CMAKE_BINARY_DIR}/target/${ARCH_DIR}/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "${CMAKE_BINARY_DIR}/target/${ARCH_DIR}/release/libdeltachat.${DYNAMIC_EXT}" DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(FILES "${CMAKE_BINARY_DIR}/target/${ARCH_DIR}/release/pkgconfig/deltachat.pc" DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)

Cargo.toml

@@ -1,9 +1,10 @@
 [package]
 name = "deltachat"
-version = "1.132.0"
+version = "1.142.6"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.70"
+rust-version = "1.77"
+repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [profile.dev]
 debug = 0
@@ -30,88 +31,86 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 
-[patch.crates-io]
-imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
-
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+deltachat-time = { path = "./deltachat-time" }
+deltachat-contact-tools = { workspace = true }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
-anyhow = "1"
-async-channel = "2.0.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
+anyhow = { workspace = true }
+async-broadcast = "0.7.1"
+async-channel = { workspace = true }
+async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
-backtrace = "0.3"
-base64 = "0.21"
-brotli = { version = "3.4", default-features=false, features = ["std"] }
-chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
+base64 = { workspace = true }
+brotli = { version = "6", default-features=false, features = ["std"] }
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
-futures = "0.3"
-futures-lite = "2.0.0"
+futures = { workspace = true }
+futures-lite = { workspace = true }
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { git = "https://github.com/n0-computer/iroh", branch = "maint-0.4", default-features = false }
-kamadak-exif = "0.5"
+image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh_old = { version = "0.4.2", default-features = false, package = "iroh"}
+iroh-net = { version = "0.22.0", default-features = false }
+iroh-gossip = { version = "0.22.0", default-features = false, features = ["net"] }
+kamadak-exif = "0.5.3"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
-mailparse = "0.14"
+libc = { workspace = true }
+mailparse = "0.15"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
-num-traits = "0.2"
-once_cell = "1.18.0"
+num-traits = { workspace = true }
+once_cell = { workspace = true }
 percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.10", default-features = false }
-pin-project = "1"
-pretty_env_logger = { version = "0.5", optional = true }
+pgp = { version = "0.13", default-features = false }
 qrcodegen = "1.7.0"
-quick-xml = "0.31"
-rand = "0.8"
-regex = "1.9"
-reqwest = { version = "0.11.20", features = ["json"] }
-rusqlite = { version = "0.30", features = ["sqlcipher"] }
+quick-xml = "0.36"
+quoted_printable = "0.5"
+rand = { workspace = true }
+regex = { workspace = true }
+reqwest = { version = "0.12.5", features = ["json"] }
+rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-sanitize-filename = "0.5"
-serde_json = "1.0"
-serde = { version = "1.0", features = ["derive"] }
+sanitize-filename = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
 sha-1 = "0.10"
-sha2 = "0.10"
-smallvec = "1"
-strum = "0.25"
-strum_macros = "0.25"
+smallvec = "1.13.2"
+strum = "0.26"
+strum_macros = "0.26"
 tagger = "4.3.4"
-textwrap = "0.16.0"
-thiserror = "1"
-tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+textwrap = "0.16.1"
+thiserror = { workspace = true }
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
-tokio-stream = { version = "0.1.14", features = ["fs"] }
+tokio-stream = { version = "0.1.15", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
-tokio-util = "0.7.9"
+tokio-util = { workspace = true }
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
-ansi_term = "0.12.0"
-anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
+ansi_term = { workspace = true }
+anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "2.0.0"
-log = "0.4"
-pretty_env_logger = "0.5"
+futures-lite = { workspace = true }
+log = { workspace = true }
 proptest = { version = "1", default-features = false, features = ["std"] }
-tempfile = "3"
+tempfile = { workspace = true }
 testdir = "0.9.0"
-tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 pretty_assertions = "1.3.0"
 
 [workspace]
@@ -122,7 +121,9 @@ members = [
   "deltachat-rpc-server",
   "deltachat-ratelimit",
   "deltachat-repl",
+  "deltachat-time",
   "format-flowed",
+  "deltachat-contact-tools",
 ]
 
 [[bench]]
@@ -153,6 +154,41 @@ harness = false
 name = "send_events"
 harness = false
 
+[workspace.dependencies]
+anyhow = "1"
+ansi_term = "0.12.1"
+async-channel = "2.3.1"
+base64 = "0.22"
+chrono = { version = "0.4.38", default-features = false }
+deltachat-contact-tools = { path = "deltachat-contact-tools" }
+deltachat-jsonrpc = { path = "deltachat-jsonrpc" }
+deltachat = { path = "." }
+futures = "0.3.30"
+futures-lite = "2.3.0"
+libc = "0.2"
+log = "0.4"
+num-traits = "0.2"
+once_cell = "1.18.0"
+rand = "0.8"
+regex = "1.10"
+rusqlite = "0.32"
+sanitize-filename = "0.5"
+serde_json = "1"
+serde = "1.0"
+tempfile = "3.10.1"
+thiserror = "1"
+
+# 1.38 is the latest version before `mio` dependency update
+# that broke compilation with Android NDK r23c and r24.
+# Version 1.39.0 cannot be compiled using these NDKs,
+# see issue <https://github.com/tokio-rs/tokio/issues/6748>
+# for details.
+tokio = "~1.38.1"
+
+tokio-util = "0.7.11"
+tracing-subscriber = "0.3"
+yerpc = "0.6.2"
+
 [features]
 default = ["vendored"]
 internals = []
@@ -161,3 +197,6 @@ vendored = [
   "rusqlite/bundled-sqlcipher-vendored-openssl",
   "reqwest/native-tls-vendored"
 ]
+
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }

README.md

@@ -6,6 +6,9 @@
   <a href="https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml">
     <img alt="Rust CI" src="https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg">
   </a>
+  <a href="https://deps.rs/repo/github/deltachat/deltachat-core-rust">
+    <img alt="dependency status" src="https://deps.rs/repo/github/deltachat/deltachat-core-rust/status.svg">
+  </a>
 </p>
 
 <p align="center">
@@ -27,13 +30,13 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run --locked -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Optionally, install `deltachat-repl` binary with
 ```
-$ cargo install --path deltachat-repl/
+$ cargo install --locked --path deltachat-repl/
 ```
 and run as
 ```
@@ -121,7 +124,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests
@@ -192,6 +195,7 @@ or its language bindings:
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
 - [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
+- [Ubuntu Touch](https://codeberg.org/lk108/deltatouch)
 - several **Bots**
 
 [^1]: Out of date / unmaintained, if you like those languages feel free to start maintaining them. If you have questions we'll help you, please ask in the issues.

RELEASE.md

@@ -4,10 +4,10 @@ For example, to release version 1.116.0 of the core, do the following steps.
 
 1. Resolve all [blocker issues](https://github.com/deltachat/deltachat-core-rust/labels/blocker).
 
-2. Run `npm run build:core:constants` in the root of the repository
-   and commit generated `node/constants.js`, `node/events.js` and `node/lib/constants.js`.
+2. Update the changelog: `git cliff --unreleased --tag 1.116.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.116.0 -p CHANGELOG.md`.
 
-3. Update the changelog: `git cliff --unreleased --tag 1.116.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.116.0 -p CHANGELOG.md`.
+3. add a link to compare previous with current version to the end of CHANGELOG.md:
+  `[1.116.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.115.2...v1.116.0`
 
 4. Update the version by running `scripts/set_core_version.py 1.116.0`.
 

benches/contacts.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::contact::Contact;
 use deltachat::context::Context;

benches/create_account.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::path::PathBuf;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

benches/get_chat_msgs.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

benches/get_chatlist.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

benches/receive_emails.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::path::PathBuf;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

benches/search_msgs.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::path::Path;
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};

benches/send_events.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use criterion::{criterion_group, criterion_main, Criterion};
 
 use deltachat::context::Context;

cliff.toml

@@ -77,3 +77,17 @@ body = """
 """
 # remove the leading and trailing whitespace from the template
 trim = true
+footer = """
+{% for release in releases -%}
+    {% if release.version -%}
+        {% if release.previous.version -%}
+            [{{ release.version | trim_start_matches(pat="v") }}]: \
+                https://github.com/deltachat/deltachat-core-rust\
+                    /compare/{{ release.previous.version }}..{{ release.version }}
+        {% endif -%}
+    {% else -%}
+        [unreleased]: https://github.com/deltachat/deltachat-core-rust\
+            /compare/{{ release.previous.version }}..HEAD
+    {% endif -%}
+{% endfor %}
+"""

deltachat-contact-tools/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "deltachat-contact-tools"
+version = "0.0.0" # No semver-stable versioning
+edition = "2021"
+description = "Contact-related tools, like parsing vcards and sanitizing name and address. Meant for internal use in the deltachat crate."
+license = "MPL-2.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { workspace = true }
+once_cell = { workspace = true }
+regex = { workspace = true }
+rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
+chrono = { workspace = true, features = ["alloc", "clock", "std"] }
+
+[dev-dependencies]
+anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -0,0 +1,810 @@
+//! Contact-related tools, like parsing vcards and sanitizing name and address
+
+#![forbid(unsafe_code)]
+#![warn(
+    unused,
+    clippy::correctness,
+    missing_debug_implementations,
+    missing_docs,
+    clippy::all,
+    clippy::wildcard_imports,
+    clippy::needless_borrow,
+    clippy::cast_lossless,
+    clippy::unused_async,
+    clippy::explicit_iter_loop,
+    clippy::explicit_into_iter_loop,
+    clippy::cloned_instead_of_copied
+)]
+#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
+#![allow(
+    clippy::match_bool,
+    clippy::mixed_read_write_in_expression,
+    clippy::bool_assert_comparison,
+    clippy::manual_split_once,
+    clippy::format_push_string,
+    clippy::bool_to_int_with_if,
+    clippy::manual_range_contains
+)]
+
+use std::fmt;
+use std::ops::Deref;
+
+use anyhow::bail;
+use anyhow::Context as _;
+use anyhow::Result;
+use chrono::{DateTime, NaiveDateTime};
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+#[derive(Debug)]
+/// A Contact, as represented in a VCard.
+pub struct VcardContact {
+    /// The email address, vcard property `email`
+    pub addr: String,
+    /// This must be the name authorized by the contact itself, not a locally given name. Vcard
+    /// property `fn`. Can be empty, one should use `display_name()` to obtain the display name.
+    pub authname: String,
+    /// The contact's public PGP key in Base64, vcard property `key`
+    pub key: Option<String>,
+    /// The contact's profile image (=avatar) in Base64, vcard property `photo`
+    pub profile_image: Option<String>,
+    /// The timestamp when the vcard was created / last updated, vcard property `rev`
+    pub timestamp: Result<i64>,
+}
+
+impl VcardContact {
+    /// Returns the contact's display name.
+    pub fn display_name(&self) -> &str {
+        match self.authname.is_empty() {
+            false => &self.authname,
+            true => &self.addr,
+        }
+    }
+}
+
+/// Returns a vCard containing given contacts.
+///
+/// Calling [`parse_vcard()`] on the returned result is a reverse operation.
+pub fn make_vcard(contacts: &[VcardContact]) -> String {
+    fn format_timestamp(c: &VcardContact) -> Option<String> {
+        let timestamp = *c.timestamp.as_ref().ok()?;
+        let datetime = DateTime::from_timestamp(timestamp, 0)?;
+        Some(datetime.format("%Y%m%dT%H%M%SZ").to_string())
+    }
+
+    let mut res = "".to_string();
+    for c in contacts {
+        let addr = &c.addr;
+        let display_name = c.display_name();
+        res += &format!(
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             EMAIL:{addr}\n\
+             FN:{display_name}\n"
+        );
+        if let Some(key) = &c.key {
+            res += &format!("KEY:data:application/pgp-keys;base64,{key}\n");
+        }
+        if let Some(profile_image) = &c.profile_image {
+            res += &format!("PHOTO:data:image/jpeg;base64,{profile_image}\n");
+        }
+        if let Some(timestamp) = format_timestamp(c) {
+            res += &format!("REV:{timestamp}\n");
+        }
+        res += "END:VCARD\n";
+    }
+    res
+}
+
+/// Parses `VcardContact`s from a given `&str`.
+pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
+    fn remove_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
+        let start_of_s = s.get(..prefix.len())?;
+
+        if start_of_s.eq_ignore_ascii_case(prefix) {
+            s.get(prefix.len()..)
+        } else {
+            None
+        }
+    }
+    fn vcard_property<'a>(s: &'a str, property: &str) -> Option<&'a str> {
+        let remainder = remove_prefix(s, property)?;
+        // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
+        // then `remainder` is now `;TYPE=work:alice@example.com`
+
+        // Note: This doesn't handle the case where there are quotes around a colon,
+        // like `NAME;Foo="Some quoted text: that contains a colon":value`.
+        // This could be improved in the future, but for now, the parsing is good enough.
+        let (params, value) = remainder.split_once(':')?;
+        // In the example from above, `params` is now `;TYPE=work`
+        // and `value` is now `alice@example.com`
+
+        if params
+            .chars()
+            .next()
+            .filter(|c| !c.is_ascii_punctuation() || *c == '_')
+            .is_some()
+        {
+            // `s` started with `property`, but the next character after it was not punctuation,
+            // so this line's property is actually something else
+            return None;
+        }
+        Some(value)
+    }
+    fn parse_datetime(datetime: &str) -> Result<i64> {
+        // According to https://www.rfc-editor.org/rfc/rfc6350#section-4.3.5, the timestamp
+        // is in ISO.8601.2004 format. DateTime::parse_from_rfc3339() apparently parses
+        // ISO.8601, but fails to parse any of the examples given.
+        // So, instead just parse using a format string.
+
+        // Parses 19961022T140000Z, 19961022T140000-05, or 19961022T140000-0500.
+        let timestamp = match DateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S%#z") {
+            Ok(datetime) => datetime.timestamp(),
+            // Parses 19961022T140000.
+            Err(e) => match NaiveDateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S") {
+                Ok(datetime) => datetime
+                    .and_local_timezone(chrono::offset::Local)
+                    .single()
+                    .context("Could not apply local timezone to parsed date and time")?
+                    .timestamp(),
+                Err(_) => return Err(e.into()),
+            },
+        };
+        Ok(timestamp)
+    }
+
+    // Remove line folding, see https://datatracker.ietf.org/doc/html/rfc6350#section-3.2
+    static NEWLINE_AND_SPACE_OR_TAB: Lazy<Regex> = Lazy::new(|| Regex::new("\r?\n[\t ]").unwrap());
+    let unfolded_lines = NEWLINE_AND_SPACE_OR_TAB.replace_all(vcard, "");
+
+    let mut lines = unfolded_lines.lines().peekable();
+    let mut contacts = Vec::new();
+
+    while lines.peek().is_some() {
+        // Skip to the start of the vcard:
+        for line in lines.by_ref() {
+            if line.eq_ignore_ascii_case("BEGIN:VCARD") {
+                break;
+            }
+        }
+
+        let mut display_name = None;
+        let mut addr = None;
+        let mut key = None;
+        let mut photo = None;
+        let mut datetime = None;
+
+        for mut line in lines.by_ref() {
+            if let Some(remainder) = remove_prefix(line, "item1.") {
+                // Remove the group name, if the group is called "item1".
+                // If necessary, we can improve this to also remove groups that are called something different that "item1".
+                //
+                // Search "group name" at https://datatracker.ietf.org/doc/html/rfc6350 for more infos.
+                line = remainder;
+            }
+
+            if let Some(email) = vcard_property(line, "email") {
+                addr.get_or_insert(email);
+            } else if let Some(name) = vcard_property(line, "fn") {
+                display_name.get_or_insert(name);
+            } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
+                .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
+                .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
+                .or_else(|| remove_prefix(line, "KEY;PREF=1:data:application/pgp-keys;base64,"))
+            {
+                key.get_or_insert(k);
+            } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
+                .or_else(|| remove_prefix(line, "PHOTO;ENCODING=BASE64;JPEG:"))
+                .or_else(|| remove_prefix(line, "PHOTO;TYPE=JPEG;ENCODING=b:"))
+                .or_else(|| remove_prefix(line, "PHOTO;ENCODING=b;TYPE=JPEG:"))
+                .or_else(|| remove_prefix(line, "PHOTO;ENCODING=BASE64;TYPE=JPEG:"))
+                .or_else(|| remove_prefix(line, "PHOTO;TYPE=JPEG;ENCODING=BASE64:"))
+                .or_else(|| remove_prefix(line, "PHOTO:data:image/jpeg;base64,"))
+            {
+                photo.get_or_insert(p);
+            } else if let Some(rev) = vcard_property(line, "rev") {
+                datetime.get_or_insert(rev);
+            } else if line.eq_ignore_ascii_case("END:VCARD") {
+                break;
+            }
+        }
+
+        let (authname, addr) =
+            sanitize_name_and_addr(display_name.unwrap_or(""), addr.unwrap_or(""));
+
+        contacts.push(VcardContact {
+            authname,
+            addr,
+            key: key.map(|s| s.to_string()),
+            profile_image: photo.map(|s| s.to_string()),
+            timestamp: datetime
+                .context("No timestamp in vcard")
+                .and_then(parse_datetime),
+        });
+    }
+
+    contacts
+}
+
+/// Valid contact address.
+#[derive(Debug, Clone)]
+pub struct ContactAddress(String);
+
+impl Deref for ContactAddress {
+    type Target = str;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl AsRef<str> for ContactAddress {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Display for ContactAddress {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.0)
+    }
+}
+
+impl ContactAddress {
+    /// Constructs a new contact address from string,
+    /// normalizing and validating it.
+    pub fn new(s: &str) -> Result<Self> {
+        let addr = addr_normalize(s);
+        if !may_be_valid_addr(&addr) {
+            bail!("invalid address {:?}", s);
+        }
+        Ok(Self(addr.to_string()))
+    }
+}
+
+/// Allow converting [`ContactAddress`] to an SQLite type.
+impl rusqlite::types::ToSql for ContactAddress {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Text(self.0.to_string());
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+/// Takes a name and an address and sanitizes them:
+/// - Extracts a name from the addr if the addr is in form "Alice <alice@example.org>"
+/// - Removes special characters from the name, see [`sanitize_name()`]
+/// - Removes the name if it is equal to the address by setting it to ""
+pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
+    static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
+    let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+        (
+            if name.is_empty() {
+                captures.get(1).map_or("", |m| m.as_str())
+            } else {
+                name
+            },
+            captures
+                .get(2)
+                .map_or("".to_string(), |m| m.as_str().to_string()),
+        )
+    } else {
+        (name, addr.to_string())
+    };
+    let mut name = sanitize_name(name);
+
+    // If the 'display name' is just the address, remove it:
+    // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
+    // If the display name is empty, DC will just show the address when it needs a display name.
+    if name == addr {
+        name = "".to_string();
+    }
+
+    (name, addr)
+}
+
+/// Sanitizes a name.
+///
+/// - Removes newlines and trims the string
+/// - Removes quotes (come from some bad MUA implementations)
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_name(name: &str) -> String {
+    let name = sanitize_single_line(name);
+
+    match name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => name
+            .get(1..name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().to_string()),
+        _ => name.to_string(),
+    }
+}
+
+/// Sanitizes user input
+///
+/// - Removes newlines and trims the string
+/// - Removes potentially-malicious bidi characters
+pub fn sanitize_single_line(input: &str) -> String {
+    sanitize_bidi_characters(input.replace(['\n', '\r'], " ").trim())
+}
+
+const RTLO_CHARACTERS: [char; 5] = ['\u{202A}', '\u{202B}', '\u{202C}', '\u{202D}', '\u{202E}'];
+const ISOLATE_CHARACTERS: [char; 3] = ['\u{2066}', '\u{2067}', '\u{2068}'];
+const POP_ISOLATE_CHARACTER: char = '\u{2069}';
+/// Some control unicode characters can influence whether adjacent text is shown from
+/// left to right or from right to left.
+///
+/// Since user input is not supposed to influence how adjacent text looks,
+/// this function removes some of these characters.
+///
+/// Also see https://github.com/deltachat/deltachat-core-rust/issues/3479.
+pub fn sanitize_bidi_characters(input_str: &str) -> String {
+    // RTLO_CHARACTERS are apparently rarely used in practice.
+    // They can impact all following text, so, better remove them all:
+    let input_str = input_str.replace(|char| RTLO_CHARACTERS.contains(&char), "");
+
+    // If the ISOLATE characters are not ended with a POP DIRECTIONAL ISOLATE character,
+    // we regard the input as potentially malicious and simply remove all ISOLATE characters.
+    // See https://en.wikipedia.org/wiki/Bidirectional_text#Unicode_bidi_support
+    // and https://www.w3.org/International/questions/qa-bidi-unicode-controls.en
+    // for an explanation about ISOLATE characters.
+    fn isolate_characters_are_valid(input_str: &str) -> bool {
+        let mut isolate_character_nesting: i32 = 0;
+        for char in input_str.chars() {
+            if ISOLATE_CHARACTERS.contains(&char) {
+                isolate_character_nesting += 1;
+            } else if char == POP_ISOLATE_CHARACTER {
+                isolate_character_nesting -= 1;
+            }
+
+            // According to Wikipedia, 125 levels are allowed:
+            // https://en.wikipedia.org/wiki/Unicode_control_characters
+            // (although, in practice, we could also significantly lower this number)
+            if isolate_character_nesting < 0 || isolate_character_nesting > 125 {
+                return false;
+            }
+        }
+        isolate_character_nesting == 0
+    }
+
+    if isolate_characters_are_valid(&input_str) {
+        input_str
+    } else {
+        input_str.replace(
+            |char| ISOLATE_CHARACTERS.contains(&char) || POP_ISOLATE_CHARACTER == char,
+            "",
+        )
+    }
+}
+
+/// Returns false if addr is an invalid address, otherwise true.
+pub fn may_be_valid_addr(addr: &str) -> bool {
+    let res = EmailAddress::new(addr);
+    res.is_ok()
+}
+
+/// Returns address lowercased,
+/// with whitespace trimmed and `mailto:` prefix removed.
+pub fn addr_normalize(addr: &str) -> String {
+    let norm = addr.trim().to_lowercase();
+
+    if norm.starts_with("mailto:") {
+        norm.get(7..).unwrap_or(&norm).to_string()
+    } else {
+        norm
+    }
+}
+
+/// Compares two email addresses, normalizing them beforehand.
+pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
+    let norm1 = addr_normalize(addr1);
+    let norm2 = addr_normalize(addr2);
+
+    norm1 == norm2
+}
+
+///
+/// Represents an email address, right now just the `name@domain` portion.
+///
+/// # Example
+///
+/// ```
+/// use deltachat_contact_tools::EmailAddress;
+/// let email = match EmailAddress::new("someone@example.com") {
+///     Ok(addr) => addr,
+///     Err(e) => panic!("Error parsing address, error was {}", e),
+/// };
+/// assert_eq!(&email.local, "someone");
+/// assert_eq!(&email.domain, "example.com");
+/// assert_eq!(email.to_string(), "someone@example.com");
+/// ```
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct EmailAddress {
+    /// Local part of the email address.
+    pub local: String,
+
+    /// Email address domain.
+    pub domain: String,
+}
+
+impl fmt::Display for EmailAddress {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}@{}", self.local, self.domain)
+    }
+}
+
+impl EmailAddress {
+    /// Performs a dead-simple parse of an email address.
+    pub fn new(input: &str) -> Result<EmailAddress> {
+        if input.is_empty() {
+            bail!("empty string is not valid");
+        }
+        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
+
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            bail!("Email {:?} must not contain whitespaces, '>' or '<'", input);
+        }
+
+        match &parts[..] {
+            [domain, local] => {
+                if local.is_empty() {
+                    bail!("empty string is not valid for local part in {:?}", input);
+                }
+                if domain.is_empty() {
+                    bail!("missing domain after '@' in {:?}", input);
+                }
+                if domain.ends_with('.') {
+                    bail!("Domain {domain:?} should not contain the dot in the end");
+                }
+                Ok(EmailAddress {
+                    local: (*local).to_string(),
+                    domain: (*domain).to_string(),
+                })
+            }
+            _ => bail!("Email {:?} must contain '@' character", input),
+        }
+    }
+}
+
+impl rusqlite::types::ToSql for EmailAddress {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Text(self.to_string());
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use chrono::TimeZone;
+
+    use super::*;
+
+    #[test]
+    fn test_vcard_thunderbird() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN:'Alice Mueller'
+EMAIL;PREF=1:alice.mueller@posteo.de
+UID:a8083264-ca47-4be7-98a8-8ec3db1447ca
+END:VCARD
+BEGIN:VCARD
+VERSION:4.0
+FN:'bobzzz@freenet.de'
+EMAIL;PREF=1:bobzzz@freenet.de
+UID:cac4fef4-6351-4854-bbe4-9b6df857eaed
+END:VCARD
+",
+        );
+
+        assert_eq!(contacts[0].addr, "alice.mueller@posteo.de".to_string());
+        assert_eq!(contacts[0].authname, "Alice Mueller".to_string());
+        assert_eq!(contacts[0].key, None);
+        assert_eq!(contacts[0].profile_image, None);
+        assert!(contacts[0].timestamp.is_err());
+
+        assert_eq!(contacts[1].addr, "bobzzz@freenet.de".to_string());
+        assert_eq!(contacts[1].authname, "".to_string());
+        assert_eq!(contacts[1].key, None);
+        assert_eq!(contacts[1].profile_image, None);
+        assert!(contacts[1].timestamp.is_err());
+
+        assert_eq!(contacts.len(), 2);
+    }
+
+    #[test]
+    fn test_vcard_simple_example() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN:Alice Wonderland
+N:Wonderland;Alice;;;Ms.
+GENDER:W
+EMAIL;TYPE=work:alice@example.com
+KEY;TYPE=PGP;ENCODING=b:[base64-data]
+REV:20240418T184242Z
+
+END:VCARD",
+        );
+
+        assert_eq!(contacts[0].addr, "alice@example.com".to_string());
+        assert_eq!(contacts[0].authname, "Alice Wonderland".to_string());
+        assert_eq!(contacts[0].key, Some("[base64-data]".to_string()));
+        assert_eq!(contacts[0].profile_image, None);
+        assert_eq!(*contacts[0].timestamp.as_ref().unwrap(), 1713465762);
+
+        assert_eq!(contacts.len(), 1);
+    }
+
+    #[test]
+    fn test_make_and_parse_vcard() {
+        let contacts = [
+            VcardContact {
+                addr: "alice@example.org".to_string(),
+                authname: "Alice Wonderland".to_string(),
+                key: Some("[base64-data]".to_string()),
+                profile_image: Some("image in Base64".to_string()),
+                timestamp: Ok(1713465762),
+            },
+            VcardContact {
+                addr: "bob@example.com".to_string(),
+                authname: "".to_string(),
+                key: None,
+                profile_image: None,
+                timestamp: Ok(0),
+            },
+        ];
+        let items = [
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             EMAIL:alice@example.org\n\
+             FN:Alice Wonderland\n\
+             KEY:data:application/pgp-keys;base64,[base64-data]\n\
+             PHOTO:data:image/jpeg;base64,image in Base64\n\
+             REV:20240418T184242Z\n\
+             END:VCARD\n",
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             EMAIL:bob@example.com\n\
+             FN:bob@example.com\n\
+             REV:19700101T000000Z\n\
+             END:VCARD\n",
+        ];
+        let mut expected = "".to_string();
+        for len in 0..=contacts.len() {
+            let contacts = &contacts[0..len];
+            let vcard = make_vcard(contacts);
+            if len > 0 {
+                expected += items[len - 1];
+            }
+            assert_eq!(vcard, expected);
+            let parsed = parse_vcard(&vcard);
+            assert_eq!(parsed.len(), contacts.len());
+            for i in 0..parsed.len() {
+                assert_eq!(parsed[i].addr, contacts[i].addr);
+                assert_eq!(parsed[i].authname, contacts[i].authname);
+                assert_eq!(parsed[i].key, contacts[i].key);
+                assert_eq!(parsed[i].profile_image, contacts[i].profile_image);
+                assert_eq!(
+                    parsed[i].timestamp.as_ref().unwrap(),
+                    contacts[i].timestamp.as_ref().unwrap()
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_contact_address() -> Result<()> {
+        let alice_addr = "alice@example.org";
+        let contact_address = ContactAddress::new(alice_addr)?;
+        assert_eq!(contact_address.as_ref(), alice_addr);
+
+        let invalid_addr = "<> foobar";
+        assert!(ContactAddress::new(invalid_addr).is_err());
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_emailaddress_parse() {
+        assert_eq!(EmailAddress::new("").is_ok(), false);
+        assert_eq!(
+            EmailAddress::new("user@domain.tld").unwrap(),
+            EmailAddress {
+                local: "user".into(),
+                domain: "domain.tld".into(),
+            }
+        );
+        assert_eq!(
+            EmailAddress::new("user@localhost").unwrap(),
+            EmailAddress {
+                local: "user".into(),
+                domain: "localhost".into()
+            }
+        );
+        assert_eq!(EmailAddress::new("uuu").is_ok(), false);
+        assert_eq!(EmailAddress::new("dd.tt").is_ok(), false);
+        assert!(EmailAddress::new("tt.dd@uu").is_ok());
+        assert!(EmailAddress::new("u@d").is_ok());
+        assert!(EmailAddress::new("u@d.").is_err());
+        assert!(EmailAddress::new("u@d.t").is_ok());
+        assert_eq!(
+            EmailAddress::new("u@d.tt").unwrap(),
+            EmailAddress {
+                local: "u".into(),
+                domain: "d.tt".into(),
+            }
+        );
+        assert!(EmailAddress::new("u@tt").is_ok());
+        assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
+    }
+
+    #[test]
+    fn test_vcard_android() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:2.1
+N:;Bob;;;
+FN:Bob
+TEL;CELL:+1-234-567-890
+EMAIL;HOME:bob@example.org
+END:VCARD
+BEGIN:VCARD
+VERSION:2.1
+N:;Alice;;;
+FN:Alice
+EMAIL;HOME:alice@example.org
+END:VCARD
+",
+        );
+
+        assert_eq!(contacts[0].addr, "bob@example.org".to_string());
+        assert_eq!(contacts[0].authname, "Bob".to_string());
+        assert_eq!(contacts[0].key, None);
+        assert_eq!(contacts[0].profile_image, None);
+
+        assert_eq!(contacts[1].addr, "alice@example.org".to_string());
+        assert_eq!(contacts[1].authname, "Alice".to_string());
+        assert_eq!(contacts[1].key, None);
+        assert_eq!(contacts[1].profile_image, None);
+
+        assert_eq!(contacts.len(), 2);
+    }
+
+    #[test]
+    fn test_vcard_local_datetime() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             FN:Alice Wonderland\n\
+             EMAIL;TYPE=work:alice@example.org\n\
+             REV:20240418T184242\n\
+             END:VCARD",
+        );
+        assert_eq!(contacts.len(), 1);
+        assert_eq!(contacts[0].addr, "alice@example.org".to_string());
+        assert_eq!(contacts[0].authname, "Alice Wonderland".to_string());
+        assert_eq!(
+            *contacts[0].timestamp.as_ref().unwrap(),
+            chrono::offset::Local
+                .with_ymd_and_hms(2024, 4, 18, 18, 42, 42)
+                .unwrap()
+                .timestamp()
+        );
+    }
+
+    #[test]
+    fn test_vcard_with_base64_avatar() {
+        // This is not an actual base64-encoded avatar, it's just to test the parsing.
+        // This one is Android-like.
+        let vcard0 = "BEGIN:VCARD
+VERSION:2.1
+N:;Bob;;;
+FN:Bob
+EMAIL;HOME:bob@example.org
+PHOTO;ENCODING=BASE64;JPEG:/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEU
+ AAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAA
+ L8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==
+
+END:VCARD
+";
+        // This one is DOS-like.
+        let vcard1 = vcard0.replace('\n', "\r\n");
+        for vcard in [vcard0, vcard1.as_str()] {
+            let contacts = parse_vcard(vcard);
+            assert_eq!(contacts.len(), 1);
+            assert_eq!(contacts[0].addr, "bob@example.org".to_string());
+            assert_eq!(contacts[0].authname, "Bob".to_string());
+            assert_eq!(contacts[0].key, None);
+            assert_eq!(contacts[0].profile_image.as_deref().unwrap(), "/9j/4AAQSkZJRgABAQAAAQABAAD/4gIoSUNDX1BST0ZJTEUAAQEAAAIYAAAAAAQwAABtbnRyUkdCIFhZWiAAAAAAAAAAAAAAAABhY3NwAAAAAAAAAAAAAAAAL8bRuAJYoZUYrI4ZY3VWwxw4Ay28AAGBISScmf/2Q==");
+        }
+    }
+
+    #[test]
+    fn test_protonmail_vcard() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN;PREF=1:Alice Wonderland
+UID:proton-web-03747582-328d-38dc-5ddd-000000000000
+ITEM1.EMAIL;PREF=1:alice@example.org
+ITEM1.KEY;PREF=1:data:application/pgp-keys;base64,aaaaaaaaaaaaaaaaaaaaaaaaa
+ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ITEM1.KEY;PREF=2:data:application/pgp-keys;base64,bbbbbbbbbbbbbbbbbbbbbbbbb
+ bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
+ITEM1.X-PM-ENCRYPT:true
+ITEM1.X-PM-SIGN:true
+END:VCARD",
+        );
+
+        assert_eq!(contacts.len(), 1);
+        assert_eq!(&contacts[0].addr, "alice@example.org");
+        assert_eq!(&contacts[0].authname, "Alice Wonderland");
+        assert_eq!(contacts[0].key.as_ref().unwrap(), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+        assert!(contacts[0].timestamp.is_err());
+        assert_eq!(contacts[0].profile_image, None);
+    }
+
+    #[test]
+    fn test_sanitize_name() {
+        assert_eq!(&sanitize_name(" hello world   "), "hello world");
+        assert_eq!(&sanitize_name("<"), "<");
+        assert_eq!(&sanitize_name(">"), ">");
+        assert_eq!(&sanitize_name("'"), "'");
+        assert_eq!(&sanitize_name("\""), "\"");
+    }
+
+    #[test]
+    fn test_sanitize_single_line() {
+        assert_eq!(sanitize_single_line("Hi\naiae "), "Hi aiae");
+        assert_eq!(sanitize_single_line("\r\nahte\n\r"), "ahte");
+    }
+
+    #[test]
+    fn test_sanitize_bidi_characters() {
+        // Legit inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat\u{2069}"),
+            "Tes\u{2067}ting Delta Chat\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"),
+            "Tes\u{2067}ting \u{2068} Delta Chat\u{2069}\u{2069}"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"),
+            "Tes\u{2067}ting\u{2069} Delta Chat\u{2067}\u{2069}"
+        );
+
+        // Potentially-malicious inputs:
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{202C}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Testing Delta Chat\u{2069}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2067}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2069}ting Delta Chat\u{2067}"),
+            "Testing Delta Chat"
+        );
+
+        assert_eq!(
+            &sanitize_bidi_characters("Tes\u{2068}ting Delta Chat"),
+            "Testing Delta Chat"
+        );
+    }
+}

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.132.0"
+version = "1.142.6"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -14,21 +14,21 @@ name = "deltachat"
 crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
-deltachat = { path = "../", default-features = false }
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
-libc = "0.2"
-human-panic = { version = "1", default-features = false }
-num-traits = "0.2"
-serde_json = "1.0"
-tokio = { version = "1", features = ["rt-multi-thread"] }
-anyhow = "1"
-thiserror = "1"
-rand = "0.8"
-once_cell = "1.18.0"
-yerpc = { version = "0.5.1", features = ["anyhow_expose"] }
+deltachat = { workspace = true, default-features = false }
+deltachat-jsonrpc = { workspace = true, optional = true }
+libc = { workspace = true }
+human-panic = { version = "2", default-features = false }
+num-traits = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+rand = { workspace = true }
+once_cell = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose"] }
 
 [features]
 default = ["vendored"]
-vendored = ["deltachat/vendored"]
+vendored = ["deltachat/vendored", "deltachat-jsonrpc/vendored"]
 jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/DoxygenLayout.xml

@@ -9,7 +9,7 @@
       <tab type="hierarchy" visible="no" title="" intro=""/>
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="topics" visible="yes" title="Constants" intro="Here is a list of constants:"/>
     <tab type="pages" visible="yes" title="" intro=""/>
     <tab type="namespaces" visible="yes" title="">
       <tab type="namespacelist" visible="yes" title="" intro=""/>

deltachat-ffi/deltachat.h

@@ -17,7 +17,6 @@ typedef struct _dc_array     dc_array_t;
 typedef struct _dc_chatlist  dc_chatlist_t;
 typedef struct _dc_chat      dc_chat_t;
 typedef struct _dc_msg       dc_msg_t;
-typedef struct _dc_reactions dc_reactions_t;
 typedef struct _dc_contact   dc_contact_t;
 typedef struct _dc_lot       dc_lot_t;
 typedef struct _dc_provider  dc_provider_t;
@@ -363,8 +362,12 @@ uint32_t        dc_get_id                    (dc_context_t* context);
  *     Must be freed using dc_event_emitter_unref() after usage.
  *
  * Note: Use only one event emitter per context.
- * Having more than one event emitter running at the same time on the same context
- * will result in events being randomly delivered to one of the emitters.
+ * The result of having multiple event emitters is unspecified.
+ * Currently events are broadcasted to all existing event emitters,
+ * but previous versions delivered events to only one event emitter
+ * and this behavior may change again in the future.
+ * Events emitted before creation of event emitter
+ * may or may not be available to event emitter.
  */
 dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
 
@@ -406,7 +409,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `socks5_user` = SOCKS5 proxy username
  * - `socks5_password` = SOCKS5 proxy password
  * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
- * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = deprecated option, should be set to the same value as `imap_certificate_checks` but ignored by the new core
  * - `displayname`  = Own name to use when sending messages. MUAs are allowed to spread this way e.g. using CC, defaults to empty
  * - `selfstatus`   = Own status to display, e.g. in e-mail footers, defaults to empty
  * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
@@ -417,25 +420,23 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    and also recoded to a reasonable size.
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
- *                    1=send and request read receipts (default)
+ *                    1=send and request read receipts
+ *                    default=send and request read receipts, only send but not reuqest if `bot` is set
  * - `bcc_self`     = 0=do not send a copy of outgoing messages to self (default),
  *                    1=send a copy of outgoing messages to self.
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
- *                    0=do not watch the `Sent`-folder (default),
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
+ *                    0=do not watch the `Sent`-folder (default).
  * - `mvbox_move`   = 1=detect chat messages,
  *                    move them to the `DeltaChat` folder,
  *                    and watch the `DeltaChat` folder for updates (default),
  *                    0=do not move chat-messages
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `only_fetch_mvbox` = 1=Do not fetch messages from folders other than the
  *                    `DeltaChat` folder. Messages will still be fetched from the
  *                    spam folder and `sendbox_watch` will also still be respected
  *                    if enabled.
  *                    0=watch all folders normally (default)
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `show_emails`  = DC_SHOW_EMAILS_OFF (0)=
  *                    show direct replies to chats only,
  *                    DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=
@@ -481,8 +482,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `bot`          = Set to "1" if this is a bot.
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
  *                    adds Auto-Submitted header to outgoing messages,
- *                    accepts contact requests automatically (calling dc_accept_chat() is not needed for bots)
- *                    and does not cut large incoming text messages.
+ *                    accepts contact requests automatically (calling dc_accept_chat() is not needed),
+ *                    does not cut large incoming text messages,
+ *                    handles existing messages the same way as new ones if `fetch_existing_msgs=1`.
  * - `last_msg_id` = database ID of the last message processed by the bot.
  *                   This ID and IDs below it are guaranteed not to be returned
  *                   by dc_get_next_msgs() and dc_wait_next_msgs().
@@ -493,8 +495,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                   For most bots calling `dc_markseen_msgs()` is the
  *                   recommended way to update this value
  *                   even for self-sent messages.
- * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
- *                    0=do not fetch existing messages on configure.
+ * - `fetch_existing_msgs` = 0=do not fetch existing messages on configure (default),
+ *                    1=fetch most recent existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
  * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
  *                    0=use IMAP IDLE if the server supports it.
@@ -517,11 +519,20 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    0=Nothing else happens when the key changes.
  *                    1=After the key changed, `dc_chat_can_send()` returns false and `dc_chat_is_protection_broken()` returns true
  *                    until `dc_accept_chat()` is called.
+ * - `is_chatmail` = 1 if the the server is a chatmail server, 0 otherwise.
+ * - `is_muted`     = Whether a context is muted by the user.
+ *                    Muted contexts should not sound, vibrate or show notifications.
+ *                    In contrast to `dc_set_chat_mute_duration()`,
+ *                    fresh message and badge counters are not changed by this setting,
+ *                    but should be tuned down where appropriate.
  * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
  *                    The prefix should be followed by the system and maybe subsystem,
  *                    e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
  *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
  *                    however, are not handled by the core otherwise.
+ * - `webxdc_realtime_enabled` = Whether the realtime APIs should be enabled.
+ *                               0 = WebXDC realtime API is disabled and behaves as noop (default).
+ *                               1 = WebXDC realtime API is enabled.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -689,8 +700,25 @@ int             dc_get_connectivity          (dc_context_t* context);
 char*           dc_get_connectivity_html     (dc_context_t* context);
 
 
+#define DC_PUSH_NOT_CONNECTED 0
+#define DC_PUSH_HEARTBEAT     1
+#define DC_PUSH_CONNECTED     2
+
+/**
+ * Get the current push notification state.
+ * One of:
+ * - DC_PUSH_NOT_CONNECTED
+ * - DC_PUSH_HEARTBEAT
+ * - DC_PUSH_CONNECTED
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return Push notification state.
+ */
+int              dc_get_push_state           (dc_context_t* context);
+
+
 /**
- * Standalone version of dc_accounts_all_work_done().
  * Only used by the python tests.
  */
 int             dc_all_work_done             (dc_context_t* context);
@@ -1103,36 +1131,6 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
 uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
 
 
-/**
- * Send a reaction to message.
- *
- * Reaction is a string of emojis separated by spaces. Reaction to a
- * single message can be sent multiple times. The last reaction
- * received overrides all previously received reactions. It is
- * possible to remove all reactions by sending an empty string.
- *
- * @deprecated 2023-11-27, use jsonrpc method `send_reaction` instead
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_id ID of the message you react to.
- * @param reaction A string consisting of emojis separated by spaces.
- * @return The ID of the message sent out or 0 for errors.
- */
-uint32_t dc_send_reaction (dc_context_t* context, uint32_t msg_id, char *reaction);
-
-
-/**
- * Get a structure with reactions to the message.
- *
- * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
- * @memberof dc_context_t
- * @param context The context object.
- * @param msg_id The message ID to get reactions for.
- * @return A structure with all reactions to the message.
- */
-dc_reactions_t* dc_get_msg_reactions (dc_context_t *context, int msg_id);
-
-
 /**
  * A webxdc instance sends a status update to its other members.
  *
@@ -1195,6 +1193,65 @@ int dc_send_webxdc_status_update (dc_context_t* context, uint32_t msg_id, const
  */
 char* dc_get_webxdc_status_updates (dc_context_t* context, uint32_t msg_id, uint32_t serial);
 
+
+/**
+ * Set Webxdc file as integration.
+ * see dc_init_webxdc_integration() for more details about Webxdc integrations.
+ *
+ * @warning This is an experimental API which may change in the future
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param file The .xdc file to use as Webxdc integration.
+ */
+void             dc_set_webxdc_integration (dc_context_t* context, const char* file);
+
+
+/**
+ * Init a Webxdc integration.
+ *
+ * A Webxdc integration is
+ * a Webxdc showing a map, getting locations via setUpdateListener(), setting POIs via sendUpdate();
+ * core takes eg. care of feeding locations to the Webxdc or sending the data out.
+ *
+ * @warning This is an experimental API, esp. support of integration types (eg. image editor, tools) is left out for simplicity
+ *
+ * Currently, Webxdc integrations are .xdc files shipped together with the main app.
+ * Before dc_init_webxdc_integration() can be called,
+ * UI has to call dc_set_webxdc_integration() to define a .xdc file to be used as integration.
+ *
+ * dc_init_webxdc_integration() returns a Webxdc message ID that
+ * UI can open and use mostly as usual.
+ *
+ * Concrete behaviour and status updates depend on the integration, driven by UI needs.
+ *
+ * There is no need to de-initialize the integration,
+ * however, unless documented otherwise,
+ * the integration is valid only as long as not re-initialized
+ * In other words, UI must not have a Webxdc with the same integration open twice.
+ *
+ * Example:
+ *
+ * ~~~
+ * // Define a .xdc file to be used as maps integration
+ * dc_set_webxdc_integration(context, path_to_maps_xdc);
+ *
+ * // Integrate the map to a chat, the map will show locations for this chat then:
+ * uint32_t webxdc_instance = dc_init_webxdc_integration(context, any_chat_id);
+ *
+ * // Or use the Webxdc as a global map, showing locations of all chats:
+ * uint32_t webxdc_instance = dc_init_webxdc_integration(context, 0);
+ * ~~~
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param chat_id The chat to get the integration for.
+ * @return ID of the message that refers to the Webxdc instance.
+ *     UI can open a Webxdc as usual with this instance.
+ */
+uint32_t        dc_init_webxdc_integration    (dc_context_t* context, uint32_t chat_id);
+
+
 /**
  * Save a draft for a chat in the database.
  *
@@ -2448,6 +2505,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
 #define         DC_QR_FPR_WITHOUT_ADDR       230 // test1=formatted fingerprint
 #define         DC_QR_ACCOUNT                250 // text1=domain
 #define         DC_QR_BACKUP                 251
+#define         DC_QR_BACKUP2                252
 #define         DC_QR_WEBRTC_INSTANCE        260 // text1=domain, text2=instance pattern
 #define         DC_QR_ADDR                   320 // id=contact
 #define         DC_QR_TEXT                   330 // text1=text
@@ -2494,6 +2552,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr() and then dc_configure().
  *
  * - DC_QR_BACKUP:
+ * - DC_QR_BACKUP2:
  *   ask the user if they want to set up a new device.
  *   If so, pass the qr-code to dc_receive_backup().
  *
@@ -2562,7 +2621,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
- *     See https://securejoin.readthedocs.io/en/latest/new.html
+ *     See https://securejoin.delta.chat/
  *     for details about both protocols.
  * @return The text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
@@ -2598,8 +2657,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  *
  * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
- * See https://securejoin.readthedocs.io/en/latest/new.html
- * for details about both protocols.
+ * See https://securejoin.delta.chat/ for details about both protocols.
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -3084,23 +3142,6 @@ dc_context_t*  dc_accounts_get_selected_account (dc_accounts_t* accounts);
 int            dc_accounts_select_account       (dc_accounts_t* accounts, uint32_t account_id);
 
 
-/**
- * This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
- *
- * iOS can:
- * - call dc_start_io() (in case IO was not running)
- * - call dc_maybe_network()
- * - while dc_accounts_all_work_done() returns false:
- *   - Wait for #DC_EVENT_CONNECTIVITY_CHANGED
- *
- * @memberof dc_accounts_t
- * @param accounts The account manager as created by dc_accounts_new().
- * @return Whether all accounts finished their background work.
- *      #DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
- */
-int            dc_accounts_all_work_done        (dc_accounts_t* accounts);
-
-
 /**
  * Start job and IMAP/SMTP tasks for all accounts managed by the account manager.
  * If IO is already running, nothing happens.
@@ -3151,6 +3192,33 @@ void           dc_accounts_maybe_network        (dc_accounts_t* accounts);
 void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
 
 
+/**
+ * Perform a background fetch for all accounts in parallel with a timeout.
+ * Pauses the scheduler, fetches messages from imap and then resumes the scheduler.
+ *
+ * dc_accounts_background_fetch() was created for the iOS Background fetch.
+ *
+ * The `DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE` event is emitted at the end
+ * even in case of timeout, unless the function fails and returns 0.
+ * Process all events until you get this one and you can safely return to the background
+ * without forgetting to create notifications caused by timing race conditions.
+ *
+ * @memberof dc_accounts_t
+ * @param timeout The timeout in seconds
+ * @return Return 1 if DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE was emitted and 0 otherwise.
+ */
+int            dc_accounts_background_fetch    (dc_accounts_t* accounts, uint64_t timeout);
+
+
+/**
+ * Sets device token for Apple Push Notification service.
+ * Returns immediately.
+ *
+ * @memberof dc_accounts_t
+ * @param token Hexadecimal device token
+ */
+void           dc_accounts_set_push_device_token (dc_accounts_t* accounts, const char *token);
+
 /**
  * Create the event emitter that is used to receive events.
  *
@@ -4050,6 +4118,19 @@ char*           dc_msg_get_subject            (const dc_msg_t* msg);
 char*           dc_msg_get_file               (const dc_msg_t* msg);
 
 
+/**
+ * Save file copy at the user-provided path.
+ *
+ * Fails if file already exists at the provided path.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @param path Destination file path with filename and extension.
+ * @return 0 on failure, 1 on success.
+ */
+int             dc_msg_save_file              (const dc_msg_t* msg, const char* path);
+
+
 /**
  * Get an original attachment filename, with extension but without the path. To get the full path,
  * use dc_msg_get_file().
@@ -4114,7 +4195,6 @@ char*             dc_msg_get_webxdc_blob      (const dc_msg_t* msg, const char*
  *   true if the Webxdc should get full internet access, including Webrtc.
  *   currently, this is only true for encrypted Webxdc's in the self chat
  *   that have requested internet access in the manifest.
- *   this is useful for development and maybe for internal integrations at some point.
  *
  * @memberof dc_msg_t
  * @param msg The webxdc instance.
@@ -4323,9 +4403,9 @@ int             dc_msg_has_deviating_timestamp(const dc_msg_t* msg);
 
 
 /**
- * Check if a message has a location bound to it.
- * These messages are also returned by dc_get_locations()
- * and the UI may decide to display a special icon beside such messages,
+ * Check if a message has a POI location bound to it.
+ * These locations are also returned by dc_get_locations()
+ * The UI may decide to display a special icon beside such messages.
  *
  * @memberof dc_msg_t
  * @param msg The message object.
@@ -4397,6 +4477,9 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * Currently, the following types are defined:
  * - DC_INFO_PROTECTION_ENABLED (11) - Info-message for "Chat is now protected"
  * - DC_INFO_PROTECTION_DISABLED (12) - Info-message for "Chat is no longer protected"
+ * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
+ *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
+ *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4423,6 +4506,7 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_EPHEMERAL_TIMER_CHANGED   10
 #define         DC_INFO_PROTECTION_ENABLED        11
 #define         DC_INFO_PROTECTION_DISABLED       12
+#define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
 /**
@@ -5068,6 +5152,15 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
  */
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
+/**
+ * Returns whether contact is a bot.
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 0 if the contact is not a bot, 1 otherwise.
+ */
+int             dc_contact_is_bot            (dc_contact_t* contact);
+
 
 /**
  * Return the contact ID that verified a contact.
@@ -5283,52 +5376,6 @@ uint32_t        dc_lot_get_id            (const dc_lot_t* lot);
 int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 
-/**
- * @class dc_reactions_t
- * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
- *
- * An object representing all reactions for a single message.
- */
-
-/**
- * Returns array of contacts which reacted to the given message.
- *
- * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
- * @memberof dc_reactions_t
- * @param reactions The object containing message reactions.
- * @return array of contact IDs. Use dc_array_get_cnt() to get array length and
- *         dc_array_get_id() to get the IDs. Should be freed using `dc_array_unref()` after usage.
- */
-dc_array_t*     dc_reactions_get_contacts(dc_reactions_t* reactions);
-
-
-/**
- * Returns a string containing space-separated reactions of a single contact.
- *
- * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
- * @memberof dc_reactions_t
- * @param reactions The object containing message reactions.
- * @param contact_id ID of the contact.
- * @return Space-separated list of emoji sequences, which could be empty.
- *         Returned string should not be modified and should be freed
- *         with dc_str_unref() after usage.
- */
-char*           dc_reactions_get_by_contact_id(dc_reactions_t* reactions, uint32_t contact_id);
-
-
-/**
- * Frees an object containing message reactions.
- *
- * Reactions objects are created by dc_get_msg_reactions().
- *
- * @deprecated 2023-11-27
- * @memberof dc_reactions_t
- * @param reactions The object to free.
- *     If NULL is given, nothing is done.
- */
-void            dc_reactions_unref       (dc_reactions_t* reactions);
-
-
 /**
  * @defgroup DC_MSG DC_MSG
  *
@@ -5445,6 +5492,11 @@ void            dc_reactions_unref       (dc_reactions_t* reactions);
  */
 #define DC_MSG_WEBXDC    80
 
+/**
+ * Message containing shared contacts represented as a vCard (virtual contact file)
+ * with email addresses and possibly other fields.
+ */
+#define DC_MSG_VCARD     90
 
 /**
  * @}
@@ -6019,10 +6071,12 @@ void dc_event_unref(dc_event_t* event);
  * Downloading a bunch of messages just finished. This is an
  * event to allow the UI to only show one notification per message bunch,
  * instead of cluttering the user with many notifications.
- * For each of the msg_ids, an additional #DC_EVENT_INCOMING_MSG event was emitted before.
+ * UI may store #DC_EVENT_INCOMING_MSG events
+ * and display notifications for all messages at once
+ * when this event arrives.
  * 
  * @param data1 0
- * @param data2 (char*) msg_ids, a json object with the message ids.
+ * @param data2 0
  */
 #define DC_EVENT_INCOMING_MSG_BUNCH       2006
 
@@ -6206,6 +6260,18 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_SELFAVATAR_CHANGED               2110
 
 
+/**
+ * A multi-device synced config value changed. Maybe the app needs to refresh smth. For uniformity
+ * this is emitted on the source device too. The value isn't reported, otherwise it would be logged
+ * which might not be good for privacy. You can get the new value with
+ * `dc_get_config(context, data2)`.
+ *
+ * @param data1 0
+ * @param data2 (char*) Configuration key.
+ */
+#define DC_EVENT_CONFIG_SYNCED                    2111
+
+
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
@@ -6230,6 +6296,53 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
+/**
+ * Data received over an ephemeral peer channel.
+ *
+ * @param data1 (int) msg_id
+ * @param data2 (int) + (char*) binary data.
+ *     length is returned as integer with dc_event_get_data2_int()
+ *     and binary data is returned as dc_event_get_data2_str().
+ *     Binary data must be passed to dc_str_unref() afterwards.
+ */
+
+#define DC_EVENT_WEBXDC_REALTIME_DATA             2150
+
+/**
+ * Tells that the Background fetch was completed (or timed out).
+ *
+ * This event acts as a marker, when you reach this event you can be sure
+ * that all events emitted during the background fetch were processed.
+ * 
+ * This event is only emitted by the account manager
+ */
+
+#define DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE   2200
+
+/**
+ * Inform that set of chats or the order of the chats in the chatlist has changed.
+ *
+ * Sometimes this is emitted together with `DC_EVENT_CHATLIST_ITEM_CHANGED`.
+ */
+
+#define DC_EVENT_CHATLIST_CHANGED              2300
+
+/**
+ * Inform that all or a single chat list item changed and needs to be rerendered
+ * If `chat_id` is set to 0, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
+ * 
+ * @param data1 (int) chat_id chat id of chatlist item to be rerendered, if chat_id = 0 all (cached & visible) items need to be rerendered
+ */
+
+#define DC_EVENT_CHATLIST_ITEM_CHANGED         2301
+
+
+/**
+ * Inform that some events have been skipped due to event channel overflow.
+ *
+ * @param data1 (int) number of events that have been skipped
+ */
+#define DC_EVENT_CHANNEL_OVERFLOW              2400
 
 /**
  * @}
@@ -6538,12 +6651,16 @@ void dc_event_unref(dc_event_t* event);
 /// "Message opened"
 ///
 /// Used in subjects of outgoing read receipts.
+///
+/// @deprecated Deprecated 2024-07-26
 #define DC_STR_READRCPT                   31
 
 /// "The message '%1$s' you sent was displayed on the screen of the recipient."
 ///
 /// Used as message text of outgoing read receipts.
 /// - %1$s will be replaced by the subject of the displayed message
+///
+/// @deprecated Deprecated 2024-06-23
 #define DC_STR_READRCPT_MAILBODY          32
 
 /// @deprecated Deprecated, this string is no longer needed.
@@ -6998,6 +7115,8 @@ void dc_event_unref(dc_event_t* event);
 /// "You added member %1$s."
 ///
 /// Used in status messages.
+///
+/// `%1$s` will be replaced by the added member's name.
 #define DC_STR_ADD_MEMBER_BY_YOU 128
 
 /// "Member %1$s added by %2$s."
@@ -7219,6 +7338,50 @@ void dc_event_unref(dc_event_t* event);
 /// Used as the first info messages in newly created groups.
 #define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
 
+/// "Member %1$s added."
+///
+/// Used as info messages.
+///
+/// `%1$s` will be replaced by the added member's name.
+#define DC_STR_MESSAGE_ADD_MEMBER 173
+
+/// "Your email provider %1$s requires end-to-end encryption which is not setup yet."
+///
+/// Used as info messages when a message cannot be sent because it cannot be encrypted.
+///
+/// `%1$s` will be replaced by the provider's domain.
+#define DC_STR_INVALID_UNENCRYPTED_MAIL 174
+
+/// "You reacted %1$s to '%2$s'"
+///
+/// `%1$s` will be replaced by the reaction, usually an emoji
+/// `%2$s` will be replaced by the summary of the message the reaction refers to
+///
+/// Used in summaries.
+#define DC_STR_YOU_REACTED 176
+
+/// "%1$s reacted %2$s to '%3$s'"
+///
+/// `%1$s` will be replaced by the name the contact who reacted
+/// `%2$s` will be replaced by the reaction, usually an emoji
+/// `%3$s` will be replaced by the summary of the message the reaction refers to
+///
+/// Used in summaries.
+#define DC_STR_REACTED_BY 177
+
+/// "Establishing guaranteed end-to-end encryption, please wait…"
+///
+/// Used as info message.
+#define DC_STR_SECUREJOIN_WAIT 190
+
+/// "Could not yet establish guaranteed end-to-end encryption, but you may already send a message."
+///
+/// Used as info message.
+#define DC_STR_SECUREJOIN_WAIT_TIMEOUT 191
+
+/// "Contact". Deprecated, currently unused.
+#define DC_STR_CONTACT 200
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -1,10 +1,9 @@
+#![recursion_limit = "256"]
 #![warn(unused, clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
     non_upper_case_globals,
-    non_upper_case_globals,
-    non_camel_case_types,
     clippy::missing_safety_doc,
     clippy::expect_fun_call
 )]
@@ -26,15 +25,13 @@ use anyhow::Context as _;
 use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, ContactId, Origin};
-use deltachat::context::Context;
+use deltachat::context::{Context, ContextBuilder};
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
-use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
-use deltachat::stock_str::StockStrings;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
@@ -69,8 +66,6 @@ const DC_GCM_INFO_ONLY: u32 = 0x02;
 /// Struct representing the deltachat context.
 pub type dc_context_t = Context;
 
-pub type dc_reactions_t = Reactions;
-
 static RT: Lazy<Runtime> = Lazy::new(|| Runtime::new().expect("unable to create tokio runtime"));
 
 fn block_on<T>(fut: T) -> T::Output
@@ -104,12 +99,11 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            as_path(dbfile),
-            id,
-            Events::new(),
-            StockStrings::new(),
-        ))
+        block_on(
+            ContextBuilder::new(as_path(dbfile).to_path_buf())
+                .with_id(id)
+                .open(),
+        )
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -133,12 +127,11 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     }
 
     let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile),
-        id,
-        Events::new(),
-        StockStrings::new(),
-    )) {
+    match block_on(
+        ContextBuilder::new(as_path(dbfile).to_path_buf())
+            .with_id(id)
+            .build(),
+    ) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
             eprintln!("failed to create context: {err:#}");
@@ -387,7 +380,7 @@ pub unsafe extern "C" fn dc_get_connectivity(context: *const dc_context_t) -> li
         return 0;
     }
     let ctx = &*context;
-    block_on(async move { ctx.get_connectivity().await as u32 as libc::c_int })
+    block_on(ctx.get_connectivity()) as u32 as libc::c_int
 }
 
 #[no_mangle]
@@ -410,6 +403,16 @@ pub unsafe extern "C" fn dc_get_connectivity_html(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_push_state(context: *const dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_push_state()");
+        return 0;
+    }
+    let ctx = &*context;
+    block_on(ctx.push_state()) as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_all_work_done(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -556,8 +559,14 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SecurejoinJoinerProgress { .. } => 2061,
         EventType::ConnectivityChanged => 2100,
         EventType::SelfavatarChanged => 2110,
+        EventType::ConfigSynced { .. } => 2111,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
+        EventType::WebxdcRealtimeData { .. } => 2150,
+        EventType::AccountsBackgroundFetchDone => 2200,
+        EventType::ChatlistChanged => 2300,
+        EventType::ChatlistItemChanged { .. } => 2301,
+        EventType::EventChannelOverflow { .. } => 2400,
     }
 }
 
@@ -583,8 +592,11 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Error(_)
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::ErrorSelfNotInGroup(_) => 0,
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::AccountsBackgroundFetchDone => 0,
+        EventType::ChatlistChanged => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -607,8 +619,13 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SecurejoinJoinerProgress { contact_id, .. } => {
             contact_id.to_u32() as libc::c_int
         }
-        EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
-        EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcRealtimeData { msg_id, .. }
+        | EventType::WebxdcStatusUpdate { msg_id, .. }
+        | EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::ChatlistItemChanged { chat_id } => {
+            chat_id.unwrap_or_default().to_u32() as libc::c_int
+        }
+        EventType::EventChannelOverflow { n } => *n as libc::c_int,
     }
 }
 
@@ -643,8 +660,13 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::SelfavatarChanged => 0,
-        EventType::ChatModified(_) => 0,
+        | EventType::SelfavatarChanged
+        | EventType::AccountsBackgroundFetchDone
+        | EventType::ChatlistChanged
+        | EventType::ChatlistItemChanged { .. }
+        | EventType::ConfigSynced { .. }
+        | EventType::ChatModified(_)
+        | EventType::EventChannelOverflow { .. } => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
@@ -659,6 +681,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
             status_update_serial,
             ..
         } => status_update_serial.to_u32() as libc::c_int,
+        EventType::WebxdcRealtimeData { data, .. } => data.len() as libc::c_int,
     }
 }
 
@@ -705,7 +728,12 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
-        | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
+        | EventType::AccountsBackgroundFetchDone
+        | EventType::ChatEphemeralTimerModified { .. }
+        | EventType::IncomingMsgBunch { .. }
+        | EventType::ChatlistItemChanged { .. }
+        | EventType::ChatlistChanged
+        | EventType::EventChannelOverflow { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
                 comment.to_c_string().unwrap_or_default().into_raw()
@@ -717,11 +745,15 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             let data2 = file.to_c_string().unwrap_or_default();
             data2.into_raw()
         }
-        EventType::IncomingMsgBunch { msg_ids } => serde_json::to_string(msg_ids)
-            .unwrap_or_default()
-            .to_c_string()
-            .unwrap_or_default()
-            .into_raw(),
+        EventType::ConfigSynced { key } => {
+            let data2 = key.to_string().to_c_string().unwrap_or_default();
+            data2.into_raw()
+        }
+        EventType::WebxdcRealtimeData { data, .. } => {
+            let ptr = libc::malloc(data.len());
+            libc::memcpy(ptr, data.as_ptr() as *mut libc::c_void, data.len());
+            ptr as *mut libc::c_char
+        }
     }
 }
 
@@ -1001,49 +1033,6 @@ pub unsafe extern "C" fn dc_send_videochat_invitation(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_send_reaction(
-    context: *mut dc_context_t,
-    msg_id: u32,
-    reaction: *const libc::c_char,
-) -> u32 {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_send_reaction()");
-        return 0;
-    }
-    let ctx = &*context;
-
-    block_on(async move {
-        send_reaction(ctx, MsgId::new(msg_id), &to_string_lossy(reaction))
-            .await
-            .map(|msg_id| msg_id.to_u32())
-            .unwrap_or_log_default(ctx, "Failed to send reaction")
-    })
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_msg_reactions(
-    context: *mut dc_context_t,
-    msg_id: u32,
-) -> *mut dc_reactions_t {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_get_msg_reactions()");
-        return ptr::null_mut();
-    }
-    let ctx = &*context;
-
-    let reactions = if let Ok(reactions) = block_on(get_msg_reactions(ctx, MsgId::new(msg_id)))
-        .context("failed dc_get_msg_reactions() call")
-        .log_err(ctx)
-    {
-        reactions
-    } else {
-        return ptr::null_mut();
-    };
-
-    Box::into_raw(Box::new(reactions))
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_send_webxdc_status_update(
     context: *mut dc_context_t,
@@ -1087,6 +1076,43 @@ pub unsafe extern "C" fn dc_get_webxdc_status_updates(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_webxdc_integration(
+    context: *mut dc_context_t,
+    file: *const libc::c_char,
+) {
+    if context.is_null() || file.is_null() {
+        eprintln!("ignoring careless call to dc_set_webxdc_integration()");
+        return;
+    }
+    let ctx = &*context;
+    block_on(ctx.set_webxdc_integration(&to_string_lossy(file)))
+        .log_err(ctx)
+        .unwrap_or_default();
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_init_webxdc_integration(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> u32 {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_init_webxdc_integration()");
+        return 0;
+    }
+    let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
+
+    block_on(ctx.init_webxdc_integration(chat_id))
+        .log_err(ctx)
+        .map(|msg_id| msg_id.map(|id| id.to_u32()).unwrap_or_default())
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -2032,7 +2058,7 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
                     );
                     message::Message::default()
                 } else {
-                    error!(ctx, "dc_get_msg could not retrieve msg_id {msg_id}: {e:#}");
+                    warn!(ctx, "dc_get_msg could not retrieve msg_id {msg_id}: {e:#}");
                     return ptr::null_mut();
                 }
             }
@@ -3355,6 +3381,34 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha
         .unwrap_or_else(|| "".strdup())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_save_file(
+    msg: *mut dc_msg_t,
+    path: *const libc::c_char,
+) -> libc::c_int {
+    if msg.is_null() || path.is_null() {
+        eprintln!("ignoring careless call to dc_msg_save_file()");
+        return 0;
+    }
+    let ffi_msg = &*msg;
+    let ctx = &*ffi_msg.context;
+    let path = to_string_lossy(path);
+    let r = block_on(
+        ffi_msg
+            .message
+            .save_file(ctx, &std::path::PathBuf::from(path)),
+    );
+    match r {
+        Ok(()) => 1,
+        Err(_) => {
+            r.context("Failed to save file from message")
+                .log_err(ctx)
+                .unwrap_or_default();
+            0
+        }
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
@@ -4125,6 +4179,15 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_is_bot(contact: *mut dc_contact_t) -> libc::c_int {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_is_bot()");
+        return 0;
+    }
+    (*contact).contact.is_bot() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
     if contact.is_null() {
@@ -4221,45 +4284,6 @@ pub unsafe extern "C" fn dc_lot_get_timestamp(lot: *mut dc_lot_t) -> i64 {
     lot.get_timestamp()
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_reactions_get_contacts(
-    reactions: *mut dc_reactions_t,
-) -> *mut dc_array::dc_array_t {
-    if reactions.is_null() {
-        eprintln!("ignoring careless call to dc_reactions_get_contacts()");
-        return ptr::null_mut();
-    }
-
-    let reactions = &*reactions;
-    let array: dc_array_t = reactions.contacts().into();
-
-    Box::into_raw(Box::new(array))
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_reactions_get_by_contact_id(
-    reactions: *mut dc_reactions_t,
-    contact_id: u32,
-) -> *mut libc::c_char {
-    if reactions.is_null() {
-        eprintln!("ignoring careless call to dc_reactions_get_by_contact_id()");
-        return ptr::null_mut();
-    }
-
-    let reactions = &*reactions;
-    reactions.get(ContactId::new(contact_id)).as_str().strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_reactions_unref(reactions: *mut dc_reactions_t) {
-    if reactions.is_null() {
-        eprintln!("ignoring careless call to dc_reactions_unref()");
-        return;
-    }
-
-    drop(Box::from_raw(reactions));
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
@@ -4340,7 +4364,7 @@ pub unsafe extern "C" fn dc_backup_provider_wait(provider: *mut dc_backup_provid
     let ctx = &*ffi_provider.context;
     let provider = &mut ffi_provider.provider;
     block_on(provider)
-        .context("Failed to await BackupProvider")
+        .context("Failed to await backup provider")
         .log_err(ctx)
         .set_last_error(ctx)
         .ok();
@@ -4394,7 +4418,7 @@ trait ResultExt<T, E> {
     /// Like `log_err()`, but:
     /// - returns the default value instead of an Err value.
     /// - emits an error instead of a warning for an [Err] result. This means
-    /// that the error will be shown to the user in a small pop-up.
+    ///   that the error will be shown to the user in a small pop-up.
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
 }
 
@@ -4460,19 +4484,6 @@ where
     }
 }
 
-trait ResultNullableExt<T> {
-    fn into_raw(self) -> *mut T;
-}
-
-impl<T, E> ResultNullableExt<T> for Result<T, E> {
-    fn into_raw(self) -> *mut T {
-        match self {
-            Ok(t) => Box::into_raw(Box::new(t)),
-            Err(_) => ptr::null_mut(),
-        }
-    }
-}
-
 fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> Vec<MsgId> {
     let ids = unsafe { std::slice::from_raw_parts(msg_ids, msg_cnt as usize) };
     let msg_ids: Vec<MsgId> = ids
@@ -4828,16 +4839,6 @@ pub unsafe extern "C" fn dc_accounts_get_all(accounts: *mut dc_accounts_t) -> *m
     Box::into_raw(Box::new(array))
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_accounts_all_work_done(accounts: *mut dc_accounts_t) -> libc::c_int {
-    if accounts.is_null() {
-        eprintln!("ignoring careless call to dc_accounts_all_work_done()");
-        return 0;
-    }
-    let accounts = &*accounts;
-    block_on(async move { accounts.read().await.all_work_done().await as libc::c_int })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
     if accounts.is_null() {
@@ -4882,6 +4883,49 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_background_fetch(
+    accounts: *mut dc_accounts_t,
+    timeout_in_seconds: u64,
+) -> libc::c_int {
+    if accounts.is_null() || timeout_in_seconds <= 2 {
+        eprintln!("ignoring careless call to dc_accounts_background_fetch()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    block_on(async move {
+        let accounts = accounts.read().await;
+        accounts
+            .background_fetch(Duration::from_secs(timeout_in_seconds))
+            .await;
+    });
+    1
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_set_push_device_token(
+    accounts: *mut dc_accounts_t,
+    token: *const libc::c_char,
+) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_set_push_device_token()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    let token = to_string_lossy(token);
+
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        if let Err(err) = accounts.set_push_device_token(&token).await {
+            accounts.emit_event(EventType::Error(format!(
+                "Failed to set notify token: {err:#}."
+            )));
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     accounts: *mut dc_accounts_t,
@@ -4919,7 +4963,9 @@ mod jsonrpc {
         }
 
         let account_manager = &*account_manager;
-        let cmd_api = deltachat_jsonrpc::api::CommandApi::from_arc(account_manager.inner.clone());
+        let cmd_api = block_on(deltachat_jsonrpc::api::CommandApi::from_arc(
+            account_manager.inner.clone(),
+        ));
 
         let (request_handle, receiver) = RpcClient::new();
         let handle = RpcSession::new(request_handle, cmd_api);

deltachat-ffi/src/lot.rs

@@ -50,6 +50,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::Backup { .. } => None,
+                Qr::Backup2 { .. } => None,
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
                 Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
@@ -102,6 +103,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => LotState::QrFprWithoutAddr,
                 Qr::Account { .. } => LotState::QrAccount,
                 Qr::Backup { .. } => LotState::QrBackup,
+                Qr::Backup2 { .. } => LotState::QrBackup2,
                 Qr::WebrtcInstance { .. } => LotState::QrWebrtcInstance,
                 Qr::Addr { .. } => LotState::QrAddr,
                 Qr::Url { .. } => LotState::QrUrl,
@@ -127,6 +129,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::Backup { .. } => Default::default(),
+                Qr::Backup2 { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
                 Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
@@ -177,6 +180,8 @@ pub enum LotState {
 
     QrBackup = 251,
 
+    QrBackup2 = 252,
+
     /// text1=domain, text2=instance pattern
     QrWebrtcInstance = 260,
 

deltachat-jsonrpc/Cargo.toml

@@ -1,10 +1,11 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.132.0"
+version = "1.142.6"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
 license = "MPL-2.0"
+repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [[bin]]
 name = "deltachat-jsonrpc-server"
@@ -12,29 +13,30 @@ path = "src/webserver.rs"
 required-features = ["webserver"]
 
 [dependencies]
-anyhow = "1"
-deltachat = { path = ".." }
-num-traits = "0.2"
-schemars = "0.8.13"
-serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.8.0"
-log = "0.4"
-async-channel = { version = "2.0.0" }
-futures = { version = "0.3.28" }
-serde_json = "1.0.105"
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+anyhow = { workspace = true }
+deltachat = { workspace = true }
+deltachat-contact-tools = { workspace = true }
+num-traits = { workspace = true }
+schemars = "0.8.21"
+serde = { workspace = true, features = ["derive"] }
+tempfile = { workspace = true }
+log = { workspace = true }
+async-channel = { workspace = true }
+futures = { workspace = true }
+serde_json = { workspace = true }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
-tokio = { version = "1.33.0" }
-sanitize-filename = "0.5"
-walkdir = "2.3.3"
-base64 = "0.21"
+tokio = { workspace = true }
+sanitize-filename = { workspace = true }
+walkdir = "2.5.0"
+base64 = { workspace = true }
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.10.0", optional = true }
+env_logger = { version = "0.11.5", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.33.0", features = ["full", "rt-multi-thread"] }
+tokio = { workspace = true, features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/src/api.rs

@@ -1,5 +1,8 @@
 use std::collections::BTreeMap;
+use std::path::Path;
+use std::str;
 use std::sync::Arc;
+use std::time::Duration;
 use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
@@ -15,12 +18,14 @@ use deltachat::constants::DC_MSG_ID_DAYMARKER;
 use deltachat::contact::{may_be_valid_addr, Contact, ContactId, Origin};
 use deltachat::context::get_info;
 use deltachat::ephemeral::Timer;
-use deltachat::imex;
 use deltachat::location;
 use deltachat::message::get_msg_read_receipts;
 use deltachat::message::{
     self, delete_msgs, markseen_msgs, Message, MessageState, MsgId, Viewtype,
 };
+use deltachat::peer_channels::{
+    leave_webxdc_realtime, send_webxdc_realtime_advertisement, send_webxdc_realtime_data,
+};
 use deltachat::provider::get_provider_info;
 use deltachat::qr::{self, Qr};
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
@@ -28,6 +33,8 @@ use deltachat::reaction::{get_msg_reactions, send_reaction};
 use deltachat::securejoin;
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
+use deltachat::EventEmitter;
+use deltachat::{imex, info};
 use sanitize_filename::is_sanitized;
 use tokio::fs;
 use tokio::sync::{watch, Mutex, RwLock};
@@ -39,7 +46,7 @@ pub mod types;
 use num_traits::FromPrimitive;
 use types::account::Account;
 use types::chat::FullChat;
-use types::contact::ContactObject;
+use types::contact::{ContactObject, VcardContact};
 use types::events::Event;
 use types::http::HttpResponse;
 use types::message::{MessageData, MessageObject, MessageReadReceipt};
@@ -62,14 +69,14 @@ use crate::api::types::qr::QrObject;
 struct AccountState {
     /// The Qr code for current [`CommandApi::provide_backup`] call.
     ///
-    /// If there currently is a call to [`CommandApi::provide_backup`] this will be
-    /// `Pending` or `Ready`, otherwise `NoProvider`.
-    backup_provider_qr: watch::Sender<ProviderQr>,
+    /// If there is currently is a call to [`CommandApi::provide_backup`] this will be
+    /// `Some`, otherwise `None`.
+    backup_provider_qr: watch::Sender<Option<Qr>>,
 }
 
 impl Default for AccountState {
     fn default() -> Self {
-        let (tx, _rx) = watch::channel(ProviderQr::NoProvider);
+        let tx = watch::Sender::new(None);
         Self {
             backup_provider_qr: tx,
         }
@@ -80,21 +87,30 @@ impl Default for AccountState {
 pub struct CommandApi {
     pub(crate) accounts: Arc<RwLock<Accounts>>,
 
+    /// Receiver side of the event channel.
+    ///
+    /// Events from it can be received by calling `get_next_event` method.
+    event_emitter: Arc<EventEmitter>,
+
     states: Arc<Mutex<BTreeMap<u32, AccountState>>>,
 }
 
 impl CommandApi {
     pub fn new(accounts: Accounts) -> Self {
+        let event_emitter = Arc::new(accounts.get_event_emitter());
         CommandApi {
             accounts: Arc::new(RwLock::new(accounts)),
+            event_emitter,
             states: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
 
     #[allow(dead_code)]
-    pub fn from_arc(accounts: Arc<RwLock<Accounts>>) -> Self {
+    pub async fn from_arc(accounts: Arc<RwLock<Accounts>>) -> Self {
+        let event_emitter = Arc::new(accounts.read().await.get_event_emitter());
         CommandApi {
             accounts,
+            event_emitter,
             states: Arc::new(Mutex::new(BTreeMap::new())),
         }
     }
@@ -123,21 +139,13 @@ impl CommandApi {
             .with_state(account_id, |state| state.backup_provider_qr.subscribe())
             .await;
 
-        let val: ProviderQr = receiver.borrow_and_update().clone();
-        match val {
-            ProviderQr::NoProvider => bail!("No backup being provided"),
-            ProviderQr::Pending => loop {
-                if receiver.changed().await.is_err() {
-                    bail!("No backup being provided (account state dropped)");
-                }
-                let val: ProviderQr = receiver.borrow().clone();
-                match val {
-                    ProviderQr::NoProvider => bail!("No backup being provided"),
-                    ProviderQr::Pending => continue,
-                    ProviderQr::Ready(qr) => break Ok(qr),
-                };
-            },
-            ProviderQr::Ready(qr) => Ok(qr),
+        loop {
+            if let Some(qr) = receiver.borrow_and_update().clone() {
+                return Ok(qr);
+            }
+            if receiver.changed().await.is_err() {
+                bail!("No backup being provided (account state dropped)");
+            }
         }
     }
 }
@@ -165,8 +173,7 @@ impl CommandApi {
 
     /// Get the next event.
     async fn get_next_event(&self) -> Result<Event> {
-        let event_emitter = self.accounts.read().await.get_event_emitter();
-        event_emitter
+        self.event_emitter
             .recv()
             .await
             .map(|event| event.into())
@@ -181,6 +188,16 @@ impl CommandApi {
         self.accounts.write().await.add_account().await
     }
 
+    /// Imports/migrated an existing account from a database path into this account manager.
+    /// Returns the ID of new account.
+    async fn migrate_account(&self, path_to_db: String) -> Result<u32> {
+        self.accounts
+            .write()
+            .await
+            .migrate_account(std::path::PathBuf::from(path_to_db))
+            .await
+    }
+
     async fn remove_account(&self, account_id: u32) -> Result<()> {
         self.accounts
             .write()
@@ -231,13 +248,27 @@ impl CommandApi {
         Ok(())
     }
 
+    /// Performs a background fetch for all accounts in parallel with a timeout.
+    ///
+    /// The `AccountsBackgroundFetchDone` event is emitted at the end even in case of timeout.
+    /// Process all events until you get this one and you can safely return to the background
+    /// without forgetting to create notifications caused by timing race conditions.
+    async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
+        self.accounts
+            .write()
+            .await
+            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+            .await;
+        Ok(())
+    }
+
     // ---------------------------------------------
     // Methods that work on individual accounts
     // ---------------------------------------------
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -311,6 +342,16 @@ impl CommandApi {
         ctx.get_info().await
     }
 
+    async fn get_blob_dir(&self, account_id: u32) -> Result<Option<String>> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx.get_blobdir().to_str().map(|s| s.to_owned()))
+    }
+
+    async fn draft_self_report(&self, account_id: u32) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx.draft_self_report().await?.to_u32())
+    }
+
     /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
@@ -383,7 +424,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -666,7 +707,22 @@ impl CommandApi {
         ChatId::new(chat_id).get_encryption_info(&ctx).await
     }
 
-    /// Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
+    /// Get QR code text that will offer a [SecureJoin](https://securejoin.delta.chat/) invitation.
+    ///
+    /// If `chat_id` is a group chat ID, SecureJoin QR code for the group is returned.
+    /// If `chat_id` is unset, setup contact QR code is returned.
+    async fn get_chat_securejoin_qr_code(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let chat = chat_id.map(ChatId::new);
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        Ok(qr)
+    }
+
+    /// Get QR code (text and SVG) that will offer a Setup-Contact or Verified-Group invitation.
     /// The QR code is compatible to the OPENPGP4FPR format
     /// so that a basic fingerprint comparison also works e.g. with OpenKeychain.
     ///
@@ -678,8 +734,7 @@ impl CommandApi {
     ///     the Verified-Group-Invite protocol is offered in the QR code;
     ///     works for protected groups as well as for normal groups.
     ///     If not set, the Setup-Contact protocol is offered in the QR code.
-    ///     See https://securejoin.readthedocs.io/en/latest/new.html
-    ///     for details about both protocols.
+    ///     See https://securejoin.delta.chat/ for details about both protocols.
     ///
     /// return format: `[code, svg]`
     async fn get_chat_securejoin_qr_code_svg(
@@ -689,10 +744,9 @@ impl CommandApi {
     ) -> Result<(String, String)> {
         let ctx = self.get_context(account_id).await?;
         let chat = chat_id.map(ChatId::new);
-        Ok((
-            securejoin::get_securejoin_qr(&ctx, chat).await?,
-            get_securejoin_qr_svg(&ctx, chat).await?,
-        ))
+        let qr = securejoin::get_securejoin_qr(&ctx, chat).await?;
+        let svg = get_securejoin_qr_svg(&ctx, chat).await?;
+        Ok((qr, svg))
     }
 
     /// Continue a Setup-Contact or Verified-Group-Invite protocol
@@ -707,8 +761,7 @@ impl CommandApi {
     ///
     /// Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
     ///
-    /// See https://securejoin.readthedocs.io/en/latest/new.html
-    /// for details about both protocols.
+    /// See https://securejoin.delta.chat/ for details about both protocols.
     ///
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
     ///     to `check_qr()`.
@@ -896,19 +949,35 @@ impl CommandApi {
             .to_u32())
     }
 
-    // for now only text messages, because we only used text messages in desktop thusfar
+    /// Add a message to the device-chat.
+    /// Device-messages usually contain update information
+    /// and some hints that are added during the program runs, multi-device etc.
+    /// The device-message may be defined by a label;
+    /// if a message with the same label was added or skipped before,
+    /// the message is not added again, even if the message was deleted in between.
+    /// If needed, the device-chat is created before.
+    ///
+    /// Sends the `MsgsChanged` event on success.
+    ///
+    /// Setting msg to None will prevent the device message with this label from being added in the future.
     async fn add_device_message(
         &self,
         account_id: u32,
         label: String,
-        text: String,
-    ) -> Result<u32> {
+        msg: Option<MessageData>,
+    ) -> Result<Option<u32>> {
         let ctx = self.get_context(account_id).await?;
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(text);
-        let message_id =
-            deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut msg)).await?;
-        Ok(message_id.to_u32())
+        if let Some(msg) = msg {
+            let mut message = msg.create_message(&ctx).await?;
+            let message_id =
+                deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut message)).await?;
+            if !message_id.is_unset() {
+                return Ok(Some(message_id.to_u32()));
+            }
+        } else {
+            deltachat::chat::add_device_msg(&ctx, Some(&label), None).await?;
+        }
+        Ok(None)
     }
 
     ///  Mark all messages in a chat as _noticed_.
@@ -1062,9 +1131,12 @@ impl CommandApi {
             .collect::<Vec<JSONRPCMessageListItem>>())
     }
 
-    async fn get_message(&self, account_id: u32, message_id: u32) -> Result<MessageObject> {
+    async fn get_message(&self, account_id: u32, msg_id: u32) -> Result<MessageObject> {
         let ctx = self.get_context(account_id).await?;
-        MessageObject::from_message_id(&ctx, message_id).await
+        let msg_id = MsgId::new(msg_id);
+        MessageObject::from_msg_id(&ctx, msg_id)
+            .await
+            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))
     }
 
     async fn get_message_html(&self, account_id: u32, message_id: u32) -> Result<Option<String>> {
@@ -1084,7 +1156,7 @@ impl CommandApi {
         let ctx = self.get_context(account_id).await?;
         let mut messages: HashMap<u32, MessageLoadResult> = HashMap::new();
         for message_id in message_ids {
-            let message_result = MessageObject::from_message_id(&ctx, message_id).await;
+            let message_result = MessageObject::from_msg_id(&ctx, MsgId::new(message_id)).await;
             messages.insert(
                 message_id,
                 match message_result {
@@ -1387,6 +1459,51 @@ impl CommandApi {
         Ok(contact_id.map(|id| id.to_u32()))
     }
 
+    /// Parses a vCard file located at the given path. Returns contacts in their original order.
+    async fn parse_vcard(&self, path: String) -> Result<Vec<VcardContact>> {
+        let vcard = fs::read(Path::new(&path)).await?;
+        let vcard = str::from_utf8(&vcard)?;
+        Ok(deltachat_contact_tools::parse_vcard(vcard)
+            .into_iter()
+            .map(|c| c.into())
+            .collect())
+    }
+
+    /// Imports contacts from a vCard file located at the given path.
+    ///
+    /// Returns the ids of created/modified contacts in the order they appear in the vCard.
+    async fn import_vcard(&self, account_id: u32, path: String) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let vcard = tokio::fs::read(Path::new(&path)).await?;
+        let vcard = str::from_utf8(&vcard)?;
+        Ok(deltachat::contact::import_vcard(&ctx, vcard)
+            .await?
+            .into_iter()
+            .map(|c| c.to_u32())
+            .collect())
+    }
+
+    /// Returns a vCard containing contacts with the given ids.
+    async fn make_vcard(&self, account_id: u32, contacts: Vec<u32>) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts: Vec<_> = contacts.iter().map(|&c| ContactId::new(c)).collect();
+        deltachat::contact::make_vcard(&ctx, &contacts).await
+    }
+
+    /// Sets vCard containing the given contacts to the message draft.
+    async fn set_draft_vcard(
+        &self,
+        account_id: u32,
+        msg_id: u32,
+        contacts: Vec<u32>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts: Vec<_> = contacts.iter().map(|&c| ContactId::new(c)).collect();
+        let mut msg = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
+        msg.make_vcard(&ctx, &contacts).await?;
+        msg.get_chat_id().set_draft(&ctx, Some(&mut msg)).await
+    }
+
     // ---------------------------------------------
     //                   chat
     // ---------------------------------------------
@@ -1531,20 +1648,21 @@ impl CommandApi {
     /// Returns once a remote device has retrieved the backup, or is cancelled.
     async fn provide_backup(&self, account_id: u32) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        self.with_state(account_id, |state| {
-            state.backup_provider_qr.send_replace(ProviderQr::Pending);
-        })
-        .await;
 
         let provider = imex::BackupProvider::prepare(&ctx).await?;
         self.with_state(account_id, |state| {
-            state
-                .backup_provider_qr
-                .send_replace(ProviderQr::Ready(provider.qr()));
+            state.backup_provider_qr.send_replace(Some(provider.qr()));
         })
         .await;
 
-        provider.await
+        let res = provider.await;
+
+        self.with_state(account_id, |state| {
+            state.backup_provider_qr.send_replace(None);
+        })
+        .await;
+
+        res
     }
 
     /// Returns the text of the QR code for the running [`CommandApi::provide_backup`].
@@ -1552,11 +1670,17 @@ impl CommandApi {
     /// This QR code text can be used in [`CommandApi::get_backup`] on a second device to
     /// retrieve the backup and setup this second device.
     ///
-    /// This call will fail if there is currently no concurrent call to
-    /// [`CommandApi::provide_backup`].  This call may block if the QR code is not yet
-    /// ready.
+    /// This call will block until the QR code is ready,
+    /// even if there is no concurrent call to [`CommandApi::provide_backup`],
+    /// but will fail after 60 seconds to avoid deadlocks.
     async fn get_backup_qr(&self, account_id: u32) -> Result<String> {
-        let qr = self.inner_get_backup_qr(account_id).await?;
+        let qr = tokio::time::timeout(
+            Duration::from_secs(60),
+            self.inner_get_backup_qr(account_id),
+        )
+        .await
+        .context("Backup provider did not start in time")?
+        .context("Failed to get backup QR code")?;
         qr::format_backup(&qr)
     }
 
@@ -1565,14 +1689,20 @@ impl CommandApi {
     /// This QR code can be used in [`CommandApi::get_backup`] on a second device to
     /// retrieve the backup and setup this second device.
     ///
-    /// This call will fail if there is currently no concurrent call to
-    /// [`CommandApi::provide_backup`].  This call may block if the QR code is not yet
-    /// ready.
+    /// This call will block until the QR code is ready,
+    /// even if there is no concurrent call to [`CommandApi::provide_backup`],
+    /// but will fail after 60 seconds to avoid deadlocks.
     ///
     /// Returns the QR code rendered as an SVG image.
     async fn get_backup_qr_svg(&self, account_id: u32) -> Result<String> {
         let ctx = self.get_context(account_id).await?;
-        let qr = self.inner_get_backup_qr(account_id).await?;
+        let qr = tokio::time::timeout(
+            Duration::from_secs(60),
+            self.inner_get_backup_qr(account_id),
+        )
+        .await
+        .context("Backup provider did not start in time")?
+        .context("Failed to get backup QR code")?;
         generate_backup_qr(&ctx, &qr).await
     }
 
@@ -1582,6 +1712,9 @@ impl CommandApi {
     /// the current device.
     ///
     /// Can be cancelled by stopping the ongoing process.
+    ///
+    /// Do not forget to call start_io on the account after a successful import,
+    /// otherwise it will not connect to the email server.
     async fn get_backup(&self, account_id: u32, qr_text: String) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
         let qr = qr::check_qr(&ctx, &qr_text).await?;
@@ -1675,6 +1808,37 @@ impl CommandApi {
             .await
     }
 
+    async fn send_webxdc_realtime_data(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+        data: Vec<u8>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        send_webxdc_realtime_data(&ctx, MsgId::new(instance_msg_id), data).await
+    }
+
+    async fn send_webxdc_realtime_advertisement(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let fut = send_webxdc_realtime_advertisement(&ctx, MsgId::new(instance_msg_id)).await?;
+        if let Some(fut) = fut {
+            tokio::spawn(async move {
+                fut.await.ok();
+                info!(ctx, "send_webxdc_realtime_advertisement done")
+            });
+        }
+        Ok(())
+    }
+
+    async fn leave_webxdc_realtime(&self, account_id: u32, instance_message_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        leave_webxdc_realtime(&ctx, MsgId::new(instance_message_id)).await
+    }
+
     async fn get_webxdc_status_updates(
         &self,
         account_id: u32,
@@ -1716,6 +1880,29 @@ impl CommandApi {
         Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
     }
 
+    /// Sets Webxdc file as integration.
+    /// `file` is the .xdc to use as Webxdc integration.
+    async fn set_webxdc_integration(&self, account_id: u32, file_path: String) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.set_webxdc_integration(&file_path).await
+    }
+
+    /// Returns Webxdc instance used for optional integrations.
+    /// UI can open the Webxdc as usual.
+    /// Returns `None` if there is no integration; the caller can add one using `set_webxdc_integration` then.
+    /// `integrate_for` is the chat to get the integration for.
+    async fn init_webxdc_integration(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<Option<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx
+            .init_webxdc_integration(chat_id.map(ChatId::new))
+            .await?
+            .map(|msg_id| msg_id.to_u32()))
+    }
+
     /// Makes an HTTP GET request and returns a response.
     ///
     /// `url` is the HTTP or HTTPS URL.
@@ -1808,38 +1995,7 @@ impl CommandApi {
 
     async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let mut message = Message::new(if let Some(viewtype) = data.viewtype {
-            viewtype.into()
-        } else if data.file.is_some() {
-            Viewtype::File
-        } else {
-            Viewtype::Text
-        });
-        message.set_text(data.text.unwrap_or_default());
-        if data.html.is_some() {
-            message.set_html(data.html);
-        }
-        if data.override_sender_name.is_some() {
-            message.set_override_sender_name(data.override_sender_name);
-        }
-        if let Some(file) = data.file {
-            message.set_file(file, None);
-        }
-        if let Some((latitude, longitude)) = data.location {
-            message.set_location(latitude, longitude);
-        }
-        if let Some(id) = data.quoted_message_id {
-            message
-                .set_quote(
-                    &ctx,
-                    Some(
-                        &Message::load_from_db(&ctx, MsgId::new(id))
-                            .await
-                            .context("message to quote could not be loaded")?,
-                    ),
-                )
-                .await?;
-        }
+        let mut message = data.create_message(&ctx).await?;
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
             .await?
             .to_u32();
@@ -1855,6 +2011,15 @@ impl CommandApi {
         Ok(can_send)
     }
 
+    /// Saves a file copy at the user-provided path.
+    ///
+    /// Fails if file already exists at the provided path.
+    async fn save_msg_file(&self, account_id: u32, msg_id: u32, path: String) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let message = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
+        message.save_file(&ctx, Path::new(&path)).await
+    }
+
     // ---------------------------------------------
     //           functions for the composer
     //    the composer is the message input field
@@ -1927,19 +2092,21 @@ impl CommandApi {
         );
         let destination_path = account_folder.join("stickers").join(collection);
         fs::create_dir_all(&destination_path).await?;
-        let file = message.get_file(&ctx).context("no file")?;
-        fs::copy(
-            &file,
-            destination_path.join(format!(
-                "{}.{}",
-                msg_id,
-                file.extension()
-                    .unwrap_or_default()
-                    .to_str()
-                    .unwrap_or_default()
-            )),
-        )
-        .await?;
+        let file = message.get_filename().context("no file?")?;
+        message
+            .save_file(
+                &ctx,
+                &destination_path.join(format!(
+                    "{}.{}",
+                    msg_id,
+                    Path::new(&file)
+                        .extension()
+                        .unwrap_or_default()
+                        .to_str()
+                        .unwrap_or_default()
+                )),
+            )
+            .await?;
         Ok(())
     }
 
@@ -2038,11 +2205,9 @@ impl CommandApi {
                 )
                 .await?;
         }
-        let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
-            .await?
-            .to_u32();
-        let message = MessageObject::from_message_id(&ctx, msg_id).await?;
-        Ok((msg_id, message))
+        let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message).await?;
+        let message = MessageObject::from_msg_id(&ctx, msg_id).await?;
+        Ok((msg_id.to_u32(), message))
     }
 
     // mimics the old desktop call, will get replaced with something better in the composer rewrite,
@@ -2121,13 +2286,6 @@ async fn set_config(
             value,
         )
         .await?;
-
-        match key {
-            "sentbox_watch" | "mvbox_move" | "only_fetch_mvbox" => {
-                ctx.restart_io_if_running().await;
-            }
-            _ => {}
-        }
     }
     Ok(())
 }
@@ -2143,15 +2301,3 @@ async fn get_config(
             .await
     }
 }
-
-/// Whether a QR code for a BackupProvider is currently available.
-#[allow(clippy::large_enum_variant)]
-#[derive(Clone, Debug)]
-enum ProviderQr {
-    /// There is no provider, asking for a QR is an error.
-    NoProvider,
-    /// There is a provider, the QR code is pending.
-    Pending,
-    /// There is a provider and QR code.
-    Ready(Qr),
-}

deltachat-jsonrpc/src/api/types/chat.rs

@@ -32,6 +32,7 @@ pub struct FullChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     // subtitle  - will be moved to frontend because it uses translation functions
     chat_type: u32,
     is_unpromoted: bool,
@@ -85,7 +86,7 @@ impl FullChat {
         let can_send = chat.can_send(context).await?;
 
         let was_seen_recently = if chat.get_type() == Chattype::Single {
-            match contact_ids.get(0) {
+            match contact_ids.first() {
                 Some(contact) => Contact::get_by_id(context, *contact)
                     .await
                     .context("failed to load contact for was_seen_recently")?
@@ -104,6 +105,7 @@ impl FullChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),
@@ -153,6 +155,7 @@ pub struct BasicChat {
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
+    pinned: bool,
     chat_type: u32,
     is_unpromoted: bool,
     is_self_talk: bool,
@@ -180,6 +183,7 @@ impl BasicChat {
             is_protected: chat.is_protected(),
             profile_image, //BLOBS ?
             archived: chat.get_visibility() == chat::ChatVisibility::Archived,
+            pinned: chat.get_visibility() == chat::ChatVisibility::Pinned,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,
             is_unpromoted: chat.is_unpromoted(),
             is_self_talk: chat.is_self_talk(),

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -102,7 +102,7 @@ pub(crate) async fn get_chat_list_item_by_id(
     let self_in_group = chat_contacts.contains(&ContactId::SELF);
 
     let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
-        let contact = chat_contacts.get(0);
+        let contact = chat_contacts.first();
         let was_seen_recently = match contact {
             Some(contact) => Contact::get_by_id(ctx, *contact)
                 .await

deltachat-jsonrpc/src/api/types/contact.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use deltachat::color;
 use deltachat::context::Context;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
@@ -45,6 +46,9 @@ pub struct ContactObject {
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
+
+    /// If the contact is a bot.
+    is_bot: bool,
 }
 
 impl ContactObject {
@@ -80,6 +84,39 @@ impl ContactObject {
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),
+            is_bot: contact.is_bot(),
         })
     }
 }
+
+#[derive(Clone, Serialize, TypeDef, schemars::JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct VcardContact {
+    /// Email address.
+    addr: String,
+    /// The contact's name, or the email address if no name was given.
+    display_name: String,
+    /// Public PGP key in Base64.
+    key: Option<String>,
+    /// Profile image in Base64.
+    profile_image: Option<String>,
+    /// Contact color as hex string.
+    color: String,
+    /// Last update timestamp.
+    timestamp: Option<i64>,
+}
+
+impl From<deltachat_contact_tools::VcardContact> for VcardContact {
+    fn from(vc: deltachat_contact_tools::VcardContact) -> Self {
+        let display_name = vc.display_name().to_string();
+        let color = color::str_to_color(&vc.addr.to_lowercase());
+        Self {
+            addr: vc.addr,
+            display_name,
+            key: vc.key,
+            profile_image: vc.profile_image,
+            color: color_int_to_hex_string(color),
+            timestamp: vc.timestamp.ok(),
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/events.rs

@@ -28,55 +28,37 @@ pub enum EventType {
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Info {
-        msg: String,
-    },
+    Info { msg: String },
 
     /// Emitted when SMTP connection is established and login was successful.
-    SmtpConnected {
-        msg: String,
-    },
+    SmtpConnected { msg: String },
 
     /// Emitted when IMAP connection is established and login was successful.
-    ImapConnected {
-        msg: String,
-    },
+    ImapConnected { msg: String },
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    SmtpMessageSent {
-        msg: String,
-    },
+    SmtpMessageSent { msg: String },
 
     /// Emitted when an IMAP message has been marked as deleted
-    ImapMessageDeleted {
-        msg: String,
-    },
+    ImapMessageDeleted { msg: String },
 
     /// Emitted when an IMAP message has been moved
-    ImapMessageMoved {
-        msg: String,
-    },
+    ImapMessageMoved { msg: String },
 
     /// Emitted before going into IDLE on the Inbox folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created
-    NewBlobFile {
-        file: String,
-    },
+    NewBlobFile { file: String },
 
     /// Emitted when an file in the $BLOBDIR was deleted
-    DeletedBlobFile {
-        file: String,
-    },
+    DeletedBlobFile { file: String },
 
     /// The library-user should write a warning string to the log.
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Warning {
-        msg: String,
-    },
+    Warning { msg: String },
 
     /// The library-user should report an error to the end-user.
     ///
@@ -88,18 +70,14 @@ pub enum EventType {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    Error {
-        msg: String,
-    },
+    Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// setChatName(), setChatProfileImage(),
     /// addContactToChat(), removeContactFromChat(),
     /// and messages sending functions.
-    ErrorSelfNotInGroup {
-        msg: String,
-    },
+    ErrorSelfNotInGroup { msg: String },
 
     /// Messages or chats changed.  One or more messages or chats changed for various
     /// reasons in the database:
@@ -110,10 +88,7 @@ pub enum EventType {
     /// `chatId` is set if only a single chat is affected by the changes, otherwise 0.
     /// `msgId` is set if only a single message is affected by the changes, otherwise 0.
     #[serde(rename_all = "camelCase")]
-    MsgsChanged {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgsChanged { chat_id: u32, msg_id: u32 },
 
     /// Reactions for the message changed.
     #[serde(rename_all = "camelCase")]
@@ -126,60 +101,39 @@ pub enum EventType {
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
-    /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
+    /// There is no extra #DC_EVENT_MSGS_CHANGED event sent together with this event.
     #[serde(rename_all = "camelCase")]
-    IncomingMsg {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    IncomingMsg { chat_id: u32, msg_id: u32 },
 
-    /// Downloading a bunch of messages just finished. This is an experimental
+    /// Downloading a bunch of messages just finished. This is an
     /// event to allow the UI to only show one notification per message bunch,
     /// instead of cluttering the user with many notifications.
-    ///
-    /// msg_ids contains the message ids.
     #[serde(rename_all = "camelCase")]
-    IncomingMsgBunch {
-        msg_ids: Vec<u32>,
-    },
+    IncomingMsgBunch,
 
     /// Messages were seen or noticed.
     /// chat id is always set.
     #[serde(rename_all = "camelCase")]
-    MsgsNoticed {
-        chat_id: u32,
-    },
+    MsgsNoticed { chat_id: u32 },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgDelivered {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDelivered { chat_id: u32, msg_id: u32 },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgFailed {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgFailed { chat_id: u32, msg_id: u32 },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgRead {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgRead { chat_id: u32, msg_id: u32 },
 
     /// A single message is deleted.
     #[serde(rename_all = "camelCase")]
-    MsgDeleted {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDeleted { chat_id: u32, msg_id: u32 },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -189,24 +143,17 @@ pub enum EventType {
     /// This event does not include ephemeral timer modification, which
     /// is a separate event.
     #[serde(rename_all = "camelCase")]
-    ChatModified {
-        chat_id: u32,
-    },
+    ChatModified { chat_id: u32 },
 
     /// Chat ephemeral timer changed.
     #[serde(rename_all = "camelCase")]
-    ChatEphemeralTimerModified {
-        chat_id: u32,
-        timer: u32,
-    },
+    ChatEphemeralTimerModified { chat_id: u32, timer: u32 },
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[serde(rename_all = "camelCase")]
-    ContactsChanged {
-        contact_id: Option<u32>,
-    },
+    ContactsChanged { contact_id: Option<u32> },
 
     /// Location of one or more contact has changed.
     ///
@@ -214,9 +161,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     this parameter is set to `None`.
     #[serde(rename_all = "camelCase")]
-    LocationChanged {
-        contact_id: Option<u32>,
-    },
+    LocationChanged { contact_id: Option<u32> },
 
     /// Inform about the configuration progress started by configure().
     ConfigureProgress {
@@ -234,9 +179,7 @@ pub enum EventType {
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexProgress {
-        progress: usize,
-    },
+    ImexProgress { progress: usize },
 
     /// A file has been exported. A file has been written by imex().
     /// This event may be sent multiple times by a single call to imex().
@@ -246,9 +189,7 @@ pub enum EventType {
     ///
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexFileWritten {
-        path: String,
-    },
+    ImexFileWritten { path: String },
 
     /// Progress information of a secure-join handshake from the view of the inviter
     /// (Alice, the person who shows the QR code).
@@ -263,10 +204,7 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[serde(rename_all = "camelCase")]
-    SecurejoinInviterProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -277,10 +215,7 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[serde(rename_all = "camelCase")]
-    SecurejoinJoinerProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -288,19 +223,49 @@ pub enum EventType {
     /// getConnectivityHtml() for details.
     ConnectivityChanged,
 
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: String,
+    },
+
     #[serde(rename_all = "camelCase")]
     WebxdcStatusUpdate {
         msg_id: u32,
         status_update_serial: u32,
     },
 
+    /// Data received over an ephemeral peer channel.
+    #[serde(rename_all = "camelCase")]
+    WebxdcRealtimeData { msg_id: u32, data: Vec<u8> },
+
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
-    WebxdcInstanceDeleted {
-        msg_id: u32,
-    },
+    WebxdcInstanceDeleted { msg_id: u32 },
+
+    /// Tells that the Background fetch was completed (or timed out).
+    /// This event acts as a marker, when you reach this event you can be sure
+    /// that all events emitted during the background fetch were processed.
+    ///
+    /// This event is only emitted by the account manager
+    AccountsBackgroundFetchDone,
+    /// Inform that set of chats or the order of the chats in the chatlist has changed.
+    ///
+    /// Sometimes this is emitted together with `UIChatlistItemChanged`.
+    ChatlistChanged,
+
+    /// Inform that a single chat list item changed and needs to be rerendered.
+    /// If `chat_id` is set to None, then all currently visible chats need to be rerendered, and all not-visible items need to be cleared from cache if the UI has a cache.
+    #[serde(rename_all = "camelCase")]
+    ChatlistItemChanged { chat_id: Option<u32> },
+
+    /// Inform than some events have been skipped due to event channel overflow.
+    EventChannelOverflow { n: u64 },
 }
 
 impl From<CoreEventType> for EventType {
@@ -336,9 +301,7 @@ impl From<CoreEventType> for EventType {
                 chat_id: chat_id.to_u32(),
                 msg_id: msg_id.to_u32(),
             },
-            CoreEventType::IncomingMsgBunch { msg_ids } => IncomingMsgBunch {
-                msg_ids: msg_ids.into_iter().map(|id| id.to_u32()).collect(),
-            },
+            CoreEventType::IncomingMsgBunch => IncomingMsgBunch,
             CoreEventType::MsgsNoticed(chat_id) => MsgsNoticed {
                 chat_id: chat_id.to_u32(),
             },
@@ -396,6 +359,9 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ConnectivityChanged => ConnectivityChanged,
             CoreEventType::SelfavatarChanged => SelfavatarChanged,
+            CoreEventType::ConfigSynced { key } => ConfigSynced {
+                key: key.to_string(),
+            },
             CoreEventType::WebxdcStatusUpdate {
                 msg_id,
                 status_update_serial,
@@ -403,9 +369,19 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 status_update_serial: status_update_serial.to_u32(),
             },
+            CoreEventType::WebxdcRealtimeData { msg_id, data } => WebxdcRealtimeData {
+                msg_id: msg_id.to_u32(),
+                data,
+            },
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
+            CoreEventType::AccountsBackgroundFetchDone => AccountsBackgroundFetchDone,
+            CoreEventType::ChatlistItemChanged { chat_id } => ChatlistItemChanged {
+                chat_id: chat_id.map(|id| id.to_u32()),
+            },
+            CoreEventType::ChatlistChanged => ChatlistChanged,
+            CoreEventType::EventChannelOverflow { n } => EventChannelOverflow { n },
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -1,3 +1,4 @@
+use crate::api::VcardContact;
 use anyhow::{Context as _, Result};
 use deltachat::chat::Chat;
 use deltachat::chat::ChatItem;
@@ -35,6 +36,10 @@ pub struct MessageObject {
     parent_id: Option<u32>,
 
     text: String,
+
+    /// Check if a message has a POI location bound to it.
+    /// These locations are also returned by `get_locations` method.
+    /// The UI may decide to display a special icon beside such messages.
     has_location: bool,
     has_html: bool,
     view_type: MessageViewtype,
@@ -83,6 +88,8 @@ pub struct MessageObject {
     download_state: DownloadState,
 
     reactions: Option<JSONRPCReactions>,
+
+    vcard_contact: Option<VcardContact>,
 }
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
@@ -105,11 +112,6 @@ enum MessageQuote {
 }
 
 impl MessageObject {
-    pub async fn from_message_id(context: &Context, message_id: u32) -> Result<Self> {
-        let msg_id = MsgId::new(message_id);
-        Self::from_msg_id(context, msg_id).await
-    }
-
     pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
         let message = Message::load_from_db(context, msg_id).await?;
 
@@ -174,6 +176,13 @@ impl MessageObject {
             Some(reactions.into())
         };
 
+        let vcard_contacts: Vec<VcardContact> = message
+            .vcard_contacts(context)
+            .await?
+            .into_iter()
+            .map(Into::into)
+            .collect();
+
         Ok(MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
@@ -233,6 +242,8 @@ impl MessageObject {
             download_state,
 
             reactions,
+
+            vcard_contact: vcard_contacts.first().cloned(),
         })
     }
 }
@@ -275,6 +286,11 @@ pub enum MessageViewtype {
 
     /// Message is an webxdc instance.
     Webxdc,
+
+    /// Message containing shared contacts represented as a vCard (virtual contact file)
+    /// with email addresses and possibly other fields.
+    /// Use `parse_vcard()` to retrieve them.
+    Vcard,
 }
 
 impl From<Viewtype> for MessageViewtype {
@@ -291,6 +307,7 @@ impl From<Viewtype> for MessageViewtype {
             Viewtype::File => MessageViewtype::File,
             Viewtype::VideochatInvitation => MessageViewtype::VideochatInvitation,
             Viewtype::Webxdc => MessageViewtype::Webxdc,
+            Viewtype::Vcard => MessageViewtype::Vcard,
         }
     }
 }
@@ -309,6 +326,7 @@ impl From<MessageViewtype> for Viewtype {
             MessageViewtype::File => Viewtype::File,
             MessageViewtype::VideochatInvitation => Viewtype::VideochatInvitation,
             MessageViewtype::Webxdc => Viewtype::Webxdc,
+            MessageViewtype::Vcard => Viewtype::Vcard,
         }
     }
 }
@@ -345,6 +363,15 @@ pub enum SystemMessageType {
     SecurejoinMessage,
     LocationStreamingEnabled,
     LocationOnly,
+    InvalidUnencryptedMail,
+
+    /// 1:1 chats info message telling that SecureJoin has started and the user should wait for it
+    /// to complete.
+    SecurejoinWait,
+
+    /// 1:1 chats info message telling that SecureJoin is still running, but the user may already
+    /// send messages.
+    SecurejoinWaitTimeout,
 
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged,
@@ -364,6 +391,9 @@ pub enum SystemMessageType {
 
     /// Webxdc info added with `info` set in `send_webxdc_status_update()`.
     WebxdcInfoMessage,
+
+    /// This message contains a users iroh node address.
+    IrohNodeAddr,
 }
 
 impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
@@ -385,6 +415,10 @@ impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
             SystemMessage::MultiDeviceSync => SystemMessageType::MultiDeviceSync,
             SystemMessage::WebxdcStatusUpdate => SystemMessageType::WebxdcStatusUpdate,
             SystemMessage::WebxdcInfoMessage => SystemMessageType::WebxdcInfoMessage,
+            SystemMessage::InvalidUnencryptedMail => SystemMessageType::InvalidUnencryptedMail,
+            SystemMessage::IrohNodeAddr => SystemMessageType::IrohNodeAddr,
+            SystemMessage::SecurejoinWait => SystemMessageType::SecurejoinWait,
+            SystemMessage::SecurejoinWaitTimeout => SystemMessageType::SecurejoinWaitTimeout,
         }
     }
 }
@@ -543,7 +577,50 @@ pub struct MessageData {
     pub file: Option<String>,
     pub location: Option<(f64, f64)>,
     pub override_sender_name: Option<String>,
+    /// Quoted message id. Takes preference over `quoted_text` (see below).
     pub quoted_message_id: Option<u32>,
+    pub quoted_text: Option<String>,
+}
+
+impl MessageData {
+    pub(crate) async fn create_message(self, context: &Context) -> Result<Message> {
+        let mut message = Message::new(if let Some(viewtype) = self.viewtype {
+            viewtype.into()
+        } else if self.file.is_some() {
+            Viewtype::File
+        } else {
+            Viewtype::Text
+        });
+        message.set_text(self.text.unwrap_or_default());
+        if self.html.is_some() {
+            message.set_html(self.html);
+        }
+        if self.override_sender_name.is_some() {
+            message.set_override_sender_name(self.override_sender_name);
+        }
+        if let Some(file) = self.file {
+            message.set_file(file, None);
+        }
+        if let Some((latitude, longitude)) = self.location {
+            message.set_location(latitude, longitude);
+        }
+        if let Some(id) = self.quoted_message_id {
+            message
+                .set_quote(
+                    context,
+                    Some(
+                        &Message::load_from_db(context, MsgId::new(id))
+                            .await
+                            .context("message to quote could not be loaded")?,
+                    ),
+                )
+                .await?;
+        } else if let Some(text) = self.quoted_text {
+            let protect = false;
+            message.set_quote_text(Some((text, protect)));
+        }
+        Ok(message)
+    }
 }
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
@@ -596,7 +673,7 @@ impl MessageInfo {
 #[derive(
     Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize, TypeDef, schemars::JsonSchema,
 )]
-#[serde(rename_all = "camelCase", tag = "variant")]
+#[serde(rename_all = "camelCase", tag = "kind")]
 pub enum EphemeralTimer {
     /// Timer is disabled.
     Disabled,

deltachat-jsonrpc/src/api/types/qr.rs

@@ -35,6 +35,11 @@ pub enum QrObject {
     Backup {
         ticket: String,
     },
+    Backup2 {
+        auth_token: String,
+
+        node_addr: String,
+    },
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
@@ -132,6 +137,14 @@ impl From<Qr> for QrObject {
             Qr::Backup { ticket } => QrObject::Backup {
                 ticket: ticket.to_string(),
             },
+            Qr::Backup2 {
+                ref node_addr,
+                auth_token,
+            } => QrObject::Backup2 {
+                node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
+
+                auth_token,
+            },
             Qr::WebrtcInstance {
                 domain,
                 instance_pattern,

deltachat-jsonrpc/src/lib.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 pub mod api;
 pub use yerpc;
 

deltachat-jsonrpc/src/webserver.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 use std::net::SocketAddr;
 use std::path::PathBuf;
 

deltachat-jsonrpc/typescript/package.json

@@ -3,7 +3,7 @@
   "dependencies": {
     "@deltachat/tiny-emitter": "3.0.0",
     "isomorphic-ws": "^4.0.1",
-    "yerpc": "^0.4.3"
+    "yerpc": "^0.6.2"
   },
   "devDependencies": {
     "@types/chai": "^4.2.21",
@@ -25,12 +25,17 @@
   "exports": {
     ".": {
       "import": "./dist/deltachat.js",
-      "require": "./dist/deltachat.cjs"
+      "require": "./dist/deltachat.cjs",
+      "types": "./dist/deltachat.d.ts"
     }
   },
   "license": "MPL-2.0",
   "main": "dist/deltachat.js",
   "name": "@deltachat/jsonrpc-client",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deltachat/deltachat-core-rust.git"
+  },
   "scripts": {
     "build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
     "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
@@ -53,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.132.0"
+  "version": "1.142.6"
 }

deltachat-repl/Cargo.toml

@@ -1,19 +1,20 @@
 [package]
 name = "deltachat-repl"
-version = "1.132.0"
+version = "1.142.6"
 license = "MPL-2.0"
 edition = "2021"
+repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [dependencies]
-ansi_term = "0.12.1"
-anyhow = "1"
-deltachat = { path = "..", features = ["internals"]}
+ansi_term = { workspace = true }
+anyhow = { workspace = true }
+deltachat = { workspace = true, features = ["internals"]}
 dirs = "5"
-log = "0.4.20"
-pretty_env_logger = "0.5"
-rusqlite = "0.30"
-rustyline = "12"
-tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+log = { workspace = true }
+rusqlite = { workspace = true }
+rustyline = "14"
+tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
 
 [features]
 default = ["vendored"]

deltachat-repl/src/cmdline.rs

@@ -3,7 +3,7 @@ extern crate dirs;
 
 use std::path::Path;
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, ensure, Result};
 use deltachat::chat::{
@@ -19,6 +19,7 @@ use deltachat::location;
 use deltachat::log::LogExt;
 use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
 use deltachat::mimeparser::SystemMessage;
+use deltachat::peer_channels::{send_webxdc_realtime_advertisement, send_webxdc_realtime_data};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::reaction::send_reaction;
@@ -33,14 +34,6 @@ use tokio::fs;
 /// e.g. bitmask 7 triggers actions defined with bits 1, 2 and 4.
 async fn reset_tables(context: &Context, bits: i32) {
     println!("Resetting tables ({bits})...");
-    if 0 != bits & 1 {
-        context
-            .sql()
-            .execute("DELETE FROM jobs;", ())
-            .await
-            .unwrap();
-        println!("(1) Jobs reset.");
-    }
     if 0 != bits & 2 {
         context
             .sql()
@@ -346,7 +339,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  receive-backup <qr>\n\
                  export-keys\n\
                  import-keys\n\
-                 export-setup\n\
                  poke [<eml-file>|<folder>|<addr> <key-file>]\n\
                  reset <flags>\n\
                  stop\n\
@@ -511,17 +503,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "import-keys" => {
             imex(&context, ImexMode::ImportSelfKeys, arg1.as_ref(), None).await?;
         }
-        "export-setup" => {
-            let setup_code = create_setup_code(&context);
-            let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = render_setup_file(&context, &setup_code).await?;
-            fs::write(&file_name, file_content).await?;
-            println!(
-                "Setup message written to: {}\nSetup code: {}",
-                file_name.display(),
-                &setup_code,
-            );
-        }
         "poke" => {
             ensure!(poke_spec(&context, Some(arg1)).await, "Poke failed");
         }
@@ -650,6 +631,30 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             println!("{cnt} chats");
             println!("{time_needed:?} to create this list");
         }
+        "start-realtime" => {
+            if arg1.is_empty() {
+                bail!("missing msgid");
+            }
+            let msg_id = MsgId::new(arg1.parse()?);
+            let res = send_webxdc_realtime_advertisement(&context, msg_id).await?;
+
+            if let Some(res) = res {
+                println!("waiting for peer channel join");
+                res.await?;
+            }
+            println!("joined peer channel");
+        }
+        "send-realtime" => {
+            if arg1.is_empty() {
+                bail!("missing msgid");
+            }
+            if arg2.is_empty() {
+                bail!("no message");
+            }
+            let msg_id = MsgId::new(arg1.parse()?);
+            send_webxdc_realtime_data(&context, msg_id, arg2.as_bytes().to_vec()).await?;
+            println!("sent realtime message");
+        }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
                 bail!("Argument [chat-id] is missing.");

deltachat-repl/src/main.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 //! This is a CLI program and a little testing frame.  This file must not be
 //! included when using Delta Chat Core as a library.
 //!
@@ -9,7 +10,6 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
@@ -20,8 +20,7 @@ use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::stock_str::StockStrings;
-use deltachat::{EventType, Events};
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
@@ -33,6 +32,7 @@ use rustyline::{
 };
 use tokio::fs;
 use tokio::runtime::Handle;
+use tracing_subscriber::EnvFilter;
 
 mod cmdline;
 use self::cmdline::*;
@@ -152,7 +152,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 14] = [
+const IMEX_COMMANDS: [&str; 13] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -163,7 +163,6 @@ const IMEX_COMMANDS: [&str; 14] = [
     "receive-backup",
     "export-keys",
     "import-keys",
-    "export-setup",
     "poke",
     "reset",
     "stop",
@@ -299,8 +298,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize) -> bool {
-        self.highlighter.highlight_char(line, pos)
+    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
+        self.highlighter.highlight_char(line, pos, forced)
     }
 }
 
@@ -312,7 +311,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]), 0, Events::new(), StockStrings::new()).await?;
+    let context = ContextBuilder::new(args[1].clone().into())
+        .with_id(1)
+        .open()
+        .await?;
 
     let events = context.get_event_emitter();
     tokio::task::spawn(async move {
@@ -401,7 +403,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    mut ctx: Context,
+    ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
@@ -481,7 +483,11 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    let _ = pretty_env_logger::try_init();
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            EnvFilter::from_default_env().add_directive("deltachat_repl=info".parse()?),
+        )
+        .init();
 
     let args = std::env::args().collect();
     start(args).await?;

deltachat-rpc-client/examples/echobot.py

@@ -4,6 +4,7 @@
 it will echo back any text send to it, it also will print to console all Delta Chat core events.
 Pass --help to the CLI to see available options.
 """
+
 from deltachat_rpc_client import events, run_bot_cli
 
 hooks = events.HookCollection()

deltachat-rpc-client/examples/echobot_advanced.py

@@ -3,6 +3,7 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
+
 import logging
 import sys
 from threading import Thread

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -2,6 +2,7 @@
 """
 Example echo bot without using hooks
 """
+
 import logging
 import sys
 

deltachat-rpc-client/pyproject.toml

@@ -1,9 +1,10 @@
 [build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+requires = ["setuptools>=45"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
+version = "1.142.6"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -19,8 +20,9 @@ classifiers = [
     "Topic :: Communications :: Chat",
     "Topic :: Communications :: Email"
 ]
-dynamic = [
-    "version"
+readme = "README.md"
+dependencies = [
+    "imap-tools",
 ]
 
 [tool.setuptools.package-data]
@@ -31,14 +33,11 @@ deltachat_rpc_client = [
 [project.entry-points.pytest11]
 "deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin"
 
-[tool.setuptools_scm]
-root = ".."
-
 [tool.black]
 line-length = 120
 
 [tool.ruff]
-select = [
+lint.select = [
   "E", "W", # pycodestyle
   "F", # Pyflakes
   "N", # pep8-naming

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -1,4 +1,5 @@
 """Delta Chat JSON-RPC high-level API"""
+
 from ._utils import AttrDict, run_bot_cli, run_client_cli
 from .account import Account
 from .chat import Chat

deltachat-rpc-client/src/deltachat_rpc_client/_utils.py

@@ -104,7 +104,11 @@ def _run_cli(
         if not client.is_configured():
             assert args.email, "Account is not configured and email must be provided"
             assert args.password, "Account is not configured and password must be provided"
-            configure_thread = Thread(run=client.configure, kwargs={"email": args.email, "password": args.password})
+            configure_thread = Thread(
+                target=client.configure,
+                daemon=True,
+                kwargs={"email": args.email, "password": args.password},
+            )
             configure_thread.start()
         client.run_forever()
 
@@ -168,3 +172,33 @@ def parse_system_add_remove(text: str) -> Optional[Tuple[str, str, str]]:
             return "removed", addr, addr
 
     return None
+
+
+class futuremethod:  # noqa: N801
+    """Decorator for async methods."""
+
+    def __init__(self, func):
+        self._func = func
+
+    def __get__(self, instance, owner=None):
+        if instance is None:
+            return self
+
+        def future(*args):
+            generator = self._func(instance, *args)
+            res = next(generator)
+
+            def f():
+                try:
+                    generator.send(res())
+                except StopIteration as e:
+                    return e.value
+
+            return f
+
+        def wrapper(*args):
+            f = future(*args)
+            return f()
+
+        wrapper.future = future
+        return wrapper

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Optional, Union
 from warnings import warn
 
-from ._utils import AttrDict
+from ._utils import AttrDict, futuremethod
 from .chat import Chat
 from .const import ChatlistFlag, ContactFlag, EventType, SpecialContactId
 from .contact import Contact
@@ -28,6 +30,10 @@ class Account:
         """Wait until the next event and return it."""
         return AttrDict(self._rpc.wait_for_event(self.id))
 
+    def clear_all_events(self):
+        """Removes all queued-up events for a given account. Useful for tests."""
+        self._rpc.clear_all_events(self.id)
+
     def remove(self) -> None:
         """Remove the account."""
         self._rpc.remove_account(self.id)
@@ -76,9 +82,24 @@ class Account:
         """Get self avatar."""
         return self.get_config("selfavatar")
 
-    def configure(self) -> None:
+    def check_qr(self, qr):
+        return self._rpc.check_qr(self.id, qr)
+
+    def set_config_from_qr(self, qr: str):
+        self._rpc.set_config_from_qr(self.id, qr)
+
+    @futuremethod
+    def configure(self):
         """Configure an account."""
-        self._rpc.configure(self.id)
+        yield self._rpc.configure.future(self.id)
+
+    def bring_online(self):
+        """Start I/O and wait until IMAP becomes IDLE."""
+        self.start_io()
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.IMAP_INBOX_IDLE:
+                break
 
     def create_contact(self, obj: Union[int, str, Contact], name: Optional[str] = None) -> Contact:
         """Create a new Contact or return an existing one.
@@ -97,6 +118,11 @@ class Account:
             obj = obj.get_snapshot().address
         return Contact(self, self._rpc.create_contact(self.id, obj, name))
 
+    def create_chat(self, account: "Account") -> Chat:
+        addr = account.get_config("addr")
+        contact = self.create_contact(addr)
+        return contact.create_chat()
+
     def get_contact_by_id(self, contact_id: int) -> Contact:
         """Return Contact instance for the given contact ID."""
         return Contact(self, contact_id)
@@ -106,7 +132,7 @@ class Account:
         contact_id = self._rpc.lookup_contact_id_by_addr(self.id, address)
         return contact_id and Contact(self, contact_id)
 
-    def get_blocked_contacts(self) -> List[AttrDict]:
+    def get_blocked_contacts(self) -> list[AttrDict]:
         """Return a list with snapshots of all blocked contacts."""
         contacts = self._rpc.get_blocked_contacts(self.id)
         return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
@@ -131,7 +157,7 @@ class Account:
         with_self: bool = False,
         verified_only: bool = False,
         snapshot: bool = False,
-    ) -> Union[List[Contact], List[AttrDict]]:
+    ) -> Union[list[Contact], list[AttrDict]]:
         """Get a filtered list of contacts.
 
         :param query: if a string is specified, only return contacts
@@ -166,7 +192,7 @@ class Account:
         no_specials: bool = False,
         alldone_hint: bool = False,
         snapshot: bool = False,
-    ) -> Union[List[Chat], List[AttrDict]]:
+    ) -> Union[list[Chat], list[AttrDict]]:
         """Return list of chats.
 
         :param query: if a string is specified only chats matching this query are returned.
@@ -218,33 +244,37 @@ class Account:
         The function returns immediately and the handshake runs in background, sending
         and receiving several messages.
         Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
-        See https://securejoin.readthedocs.io/en/latest/new.html for protocol details.
+        See https://securejoin.delta.chat/ for protocol details.
 
         :param qrdata: The text of the scanned QR code.
         """
         return Chat(self, self._rpc.secure_join(self.id, qrdata))
 
-    def get_qr_code(self) -> Tuple[str, str]:
-        """Get Setup-Contact QR Code text and SVG data.
+    def get_qr_code(self) -> str:
+        """Get Setup-Contact QR Code text.
 
-        this data needs to be transferred to another Delta Chat account
+        This data needs to be transferred to another Delta Chat account
         in a second channel, typically used by mobiles with QRcode-show + scan UX.
         """
+        return self._rpc.get_chat_securejoin_qr_code(self.id, None)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
+        """Get Setup-Contact QR code text and SVG."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.id, None)
 
     def get_message_by_id(self, msg_id: int) -> Message:
         """Return the Message instance with the given ID."""
         return Message(self, msg_id)
 
-    def mark_seen_messages(self, messages: List[Message]) -> None:
+    def mark_seen_messages(self, messages: list[Message]) -> None:
         """Mark the given set of messages as seen."""
         self._rpc.markseen_msgs(self.id, [msg.id for msg in messages])
 
-    def delete_messages(self, messages: List[Message]) -> None:
+    def delete_messages(self, messages: list[Message]) -> None:
         """Delete messages (local and remote)."""
         self._rpc.delete_messages(self.id, [msg.id for msg in messages])
 
-    def get_fresh_messages(self) -> List[Message]:
+    def get_fresh_messages(self) -> list[Message]:
         """Return the list of fresh messages, newest messages first.
 
         This call is intended for displaying notifications.
@@ -254,12 +284,12 @@ class Account:
         fresh_msg_ids = self._rpc.get_fresh_msgs(self.id)
         return [Message(self, msg_id) for msg_id in fresh_msg_ids]
 
-    def get_next_messages(self) -> List[Message]:
+    def get_next_messages(self) -> list[Message]:
         """Return a list of next messages."""
         next_msg_ids = self._rpc.get_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
-    def wait_next_messages(self) -> List[Message]:
+    def wait_next_messages(self) -> list[Message]:
         """Wait for new messages and return a list of them."""
         next_msg_ids = self._rpc.wait_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
@@ -271,6 +301,12 @@ class Account:
             if event.kind == EventType.INCOMING_MSG:
                 return event
 
+    def wait_for_incoming_msg(self):
+        """Wait for incoming message and return it.
+
+        Consumes all events before the next incoming message event."""
+        return self.get_message_by_id(self.wait_for_incoming_msg_event().msg_id)
+
     def wait_for_securejoin_inviter_success(self):
         while True:
             event = self.wait_for_event()
@@ -283,7 +319,13 @@ class Account:
             if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
                 break
 
-    def get_fresh_messages_in_arrival_order(self) -> List[Message]:
+    def wait_for_reactions_changed(self):
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.REACTIONS_CHANGED:
+                return event
+
+    def get_fresh_messages_in_arrival_order(self) -> list[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(
             "get_fresh_messages_in_arrival_order is deprecated, use get_next_messages instead.",
@@ -300,3 +342,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 import calendar
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
+from tempfile import NamedTemporaryFile
+from typing import TYPE_CHECKING, Optional, Union
 
 from ._utils import AttrDict
 from .const import ChatVisibility, ViewType
@@ -84,14 +87,20 @@ class Chat:
         self._rpc.set_chat_name(self.account.id, self.id, name)
 
     def set_ephemeral_timer(self, timer: int) -> None:
-        """Set ephemeral timer of this chat."""
+        """Set ephemeral timer of this chat in seconds.
+
+        0 means the timer is disabled, use 1 for immediate deletion."""
         self._rpc.set_chat_ephemeral_timer(self.account.id, self.id, timer)
 
     def get_encryption_info(self) -> str:
         """Return encryption info for this chat."""
         return self._rpc.get_chat_encryption_info(self.account.id, self.id)
 
-    def get_qr_code(self) -> Tuple[str, str]:
+    def get_qr_code(self) -> str:
+        """Get Join-Group QR code text."""
+        return self._rpc.get_chat_securejoin_qr_code(self.account.id, self.id)
+
+    def get_qr_code_svg(self) -> tuple[str, str]:
         """Get Join-Group QR code text and SVG data."""
         return self._rpc.get_chat_securejoin_qr_code_svg(self.account.id, self.id)
 
@@ -115,7 +124,7 @@ class Chat:
         html: Optional[str] = None,
         viewtype: Optional[ViewType] = None,
         file: Optional[str] = None,
-        location: Optional[Tuple[float, float]] = None,
+        location: Optional[tuple[float, float]] = None,
         override_sender_name: Optional[str] = None,
         quoted_msg: Optional[Union[int, Message]] = None,
     ) -> Message:
@@ -140,6 +149,10 @@ class Chat:
         msg_id = self._rpc.misc_send_text_message(self.account.id, self.id, text)
         return Message(self.account, msg_id)
 
+    def send_file(self, path):
+        """Send a file and return the resulting Message instance."""
+        return self.send_message(file=path)
+
     def send_videochat_invitation(self) -> Message:
         """Send a videochat invitation and return the resulting Message instance."""
         msg_id = self._rpc.send_videochat_invitation(self.account.id, self.id)
@@ -150,7 +163,7 @@ class Chat:
         msg_id = self._rpc.send_sticker(self.account.id, self.id, path)
         return Message(self.account, msg_id)
 
-    def forward_messages(self, messages: List[Message]) -> None:
+    def forward_messages(self, messages: list[Message]) -> None:
         """Forward a list of messages to this chat."""
         msg_ids = [msg.id for msg in messages]
         self._rpc.forward_messages(self.account.id, msg_ids, self.id)
@@ -182,7 +195,7 @@ class Chat:
         snapshot["message"] = Message(self.account, snapshot.id)
         return snapshot
 
-    def get_messages(self, info_only: bool = False, add_daymarker: bool = False) -> List[Message]:
+    def get_messages(self, info_only: bool = False, add_daymarker: bool = False) -> list[Message]:
         """get the list of messages in this chat."""
         msgs = self._rpc.get_message_ids(self.account.id, self.id, info_only, add_daymarker)
         return [Message(self.account, msg_id) for msg_id in msgs]
@@ -217,7 +230,7 @@ class Chat:
                 contact_id = cnt
             self._rpc.remove_contact_from_chat(self.account.id, self.id, contact_id)
 
-    def get_contacts(self) -> List[Contact]:
+    def get_contacts(self) -> list[Contact]:
         """Get the contacts belonging to this chat.
 
         For single/direct chats self-address is not included.
@@ -241,7 +254,7 @@ class Chat:
         contact: Optional[Contact] = None,
         timestamp_from: Optional["datetime"] = None,
         timestamp_to: Optional["datetime"] = None,
-    ) -> List[AttrDict]:
+    ) -> list[AttrDict]:
         """Get list of location snapshots for the given contact in the given timespan."""
         time_from = calendar.timegm(timestamp_from.utctimetuple()) if timestamp_from else 0
         time_to = calendar.timegm(timestamp_to.utctimetuple()) if timestamp_to else 0
@@ -249,7 +262,7 @@ class Chat:
 
         result = self._rpc.get_locations(self.account.id, self.id, contact_id, time_from, time_to)
         locations = []
-        contacts: Dict[int, Contact] = {}
+        contacts: dict[int, Contact] = {}
         for loc in result:
             location = AttrDict(loc)
             location["chat"] = self
@@ -257,3 +270,11 @@ class Chat:
             location["message"] = Message(self.account, location.msg_id)
             locations.append(location)
         return locations
+
+    def send_contact(self, contact: Contact):
+        """Send contact to the chat."""
+        vcard = contact.make_vcard()
+        with NamedTemporaryFile(suffix=".vcard") as f:
+            f.write(vcard.encode())
+            f.flush()
+            self._rpc.send_msg(self.account.id, self.id, {"viewtype": ViewType.VCARD, "file": f.name})

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -1,13 +1,13 @@
 """Event loop implementations offering high level event handling/hooking."""
+
+from __future__ import annotations
+
 import logging
 from typing import (
     TYPE_CHECKING,
     Callable,
-    Dict,
     Iterable,
     Optional,
-    Set,
-    Tuple,
     Type,
     Union,
 )
@@ -38,16 +38,16 @@ class Client:
     def __init__(
         self,
         account: "Account",
-        hooks: Optional[Iterable[Tuple[Callable, Union[type, EventFilter]]]] = None,
+        hooks: Optional[Iterable[tuple[Callable, Union[type, EventFilter]]]] = None,
         logger: Optional[logging.Logger] = None,
     ) -> None:
         self.account = account
         self.logger = logger or logging
-        self._hooks: Dict[type, Set[tuple]] = {}
+        self._hooks: dict[type, set[tuple]] = {}
         self._should_process_messages = 0
         self.add_hooks(hooks or [])
 
-    def add_hooks(self, hooks: Iterable[Tuple[Callable, Union[type, EventFilter]]]) -> None:
+    def add_hooks(self, hooks: Iterable[tuple[Callable, Union[type, EventFilter]]]) -> None:
         for hook, event in hooks:
             self.add_hook(hook, event)
 

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -59,6 +59,19 @@ class EventType(str, Enum):
     SELFAVATAR_CHANGED = "SelfavatarChanged"
     WEBXDC_STATUS_UPDATE = "WebxdcStatusUpdate"
     WEBXDC_INSTANCE_DELETED = "WebxdcInstanceDeleted"
+    CHATLIST_CHANGED = "ChatlistChanged"
+    CHATLIST_ITEM_CHANGED = "ChatlistItemChanged"
+    CONFIG_SYNCED = "ConfigSynced"
+    WEBXDC_REALTIME_DATA = "WebxdcRealtimeData"
+
+
+class ChatId(IntEnum):
+    """Special chat ids"""
+
+    TRASH = 3
+    ARCHIVED_LINK = 6
+    ALLDONE_HINT = 7
+    LAST_SPECIAL = 9
 
 
 class ChatType(IntEnum):
@@ -102,6 +115,7 @@ class ViewType(str, Enum):
     FILE = "File"
     VIDEOCHAT_INVITATION = "VideochatInvitation"
     WEBXDC = "Webxdc"
+    VCARD = "Vcard"
 
 
 class SystemMessageType(str, Enum):
@@ -122,3 +136,107 @@ class SystemMessageType(str, Enum):
     EPHEMERAL_TIMER_CHANGED = "EphemeralTimerChanged"
     MULTI_DEVICE_SYNC = "MultiDeviceSync"
     WEBXDC_INFO_MESSAGE = "WebxdcInfoMessage"
+
+
+class MessageState(IntEnum):
+    """State of the message."""
+
+    UNDEFINED = 0
+    IN_FRESH = 10
+    IN_NOTICED = 13
+    IN_SEEN = 16
+    OUT_PREPARING = 18
+    OUT_DRAFT = 19
+    OUT_PENDING = 20
+    OUT_FAILED = 24
+    OUT_DELIVERED = 26
+    OUT_MDN_RCVD = 28
+
+
+class MessageId(IntEnum):
+    """Special message ids"""
+
+    DAYMARKER = 9
+    LAST_SPECIAL = 9
+
+
+class CertificateChecks(IntEnum):
+    """Certificate checks mode"""
+
+    AUTOMATIC = 0
+    STRICT = 1
+    ACCEPT_INVALID_CERTIFICATES = 3
+
+
+class Connectivity(IntEnum):
+    """Connectivity states"""
+
+    NOT_CONNECTED = 1000
+    CONNECTING = 2000
+    WORKING = 3000
+    CONNECTED = 4000
+
+
+class KeyGenType(IntEnum):
+    """Type of the key to generate"""
+
+    DEFAULT = 0
+    RSA2048 = 1
+    ED25519 = 2
+    RSA4096 = 3
+
+
+# "Lp" means "login parameters"
+class LpAuthFlag(IntEnum):
+    """Authorization flags"""
+
+    OAUTH2 = 0x2
+    NORMAL = 0x4
+
+
+class MediaQuality(IntEnum):
+    """Media quality setting"""
+
+    BALANCED = 0
+    WORSE = 1
+
+
+class ProviderStatus(IntEnum):
+    """Provider status according to manual testing"""
+
+    OK = 1
+    PREPARATION = 2
+    BROKEN = 3
+
+
+class PushNotifyState(IntEnum):
+    """Push notifications state"""
+
+    NOT_CONNECTED = 0
+    HEARTBEAT = 1
+    CONNECTED = 2
+
+
+class ShowEmails(IntEnum):
+    """Show emails mode"""
+
+    OFF = 0
+    ACCEPTED_CONTACTS = 1
+    ALL = 2
+
+
+class SocketSecurity(IntEnum):
+    """Socket security"""
+
+    AUTOMATIC = 0
+    SSL = 1
+    STARTTLS = 2
+    PLAIN = 3
+
+
+class VideochatType(IntEnum):
+    """Video chat URL type"""
+
+    UNKNOWN = 0
+    BASICWEBRTC = 1
+    JITSI = 2

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -60,3 +60,6 @@ class Contact:
             self.account,
             self._rpc.create_chat_by_contact_id(self.account.id, self.id),
         )
+
+    def make_vcard(self) -> str:
+        return self._rpc.make_vcard(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -1,4 +1,6 @@
-from typing import TYPE_CHECKING, Dict, List
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 from ._utils import AttrDict
 from .account import Account
@@ -21,7 +23,7 @@ class DeltaChat:
         account_id = self.rpc.add_account()
         return Account(self, account_id)
 
-    def get_all_accounts(self) -> List[Account]:
+    def get_all_accounts(self) -> list[Account]:
         """Return a list of all available accounts."""
         account_ids = self.rpc.get_all_account_ids()
         return [Account(self, account_id) for account_id in account_ids]
@@ -44,6 +46,6 @@ class DeltaChat:
         """Get information about the Delta Chat core in this system."""
         return AttrDict(self.rpc.get_system_info())
 
-    def set_translations(self, translations: Dict[str, str]) -> None:
+    def set_translations(self, translations: dict[str, str]) -> None:
         """Set stock translation strings."""
         self.rpc.set_stock_strings(translations)

deltachat-rpc-client/src/deltachat_rpc_client/direct_imap.py

@@ -0,0 +1,226 @@
+"""
+Internal Python-level IMAP handling used by the tests.
+"""
+
+from __future__ import annotations
+
+import imaplib
+import io
+import pathlib
+import ssl
+from contextlib import contextmanager
+
+from imap_tools import (
+    AND,
+    Header,
+    MailBox,
+    MailBoxTls,
+    MailMessage,
+    MailMessageFlags,
+    errors,
+)
+
+from . import Account, const
+
+FLAGS = b"FLAGS"
+FETCH = b"FETCH"
+ALL = "1:*"
+
+
+class DirectImap:
+    def __init__(self, account: Account) -> None:
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        host = self.account.get_config("configured_mail_server")
+        port = int(self.account.get_config("configured_mail_port"))
+        security = int(self.account.get_config("configured_mail_security"))
+
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+
+        if security == const.SocketSecurity.PLAIN:
+            ssl_context = None
+        else:
+            ssl_context = ssl.create_default_context()
+
+            # don't check if certificate hostname doesn't match target hostname
+            ssl_context.check_hostname = False
+
+            # don't check if the certificate is trusted by a certificate authority
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+        if security == const.SocketSecurity.STARTTLS:
+            self.conn = MailBoxTls(host, port, ssl_context=ssl_context)
+        elif security == const.SocketSecurity.PLAIN or security == const.SocketSecurity.SSL:
+            self.conn = MailBox(host, port, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.logout()
+        except (OSError, imaplib.IMAP4.abort):
+            print("Could not logout direct_imap conn")
+
+    def create_folder(self, foldername):
+        try:
+            self.conn.folder.create(foldername)
+        except errors.MailboxFolderCreateError as e:
+            print("Can't create", foldername, "probably it already exists:", str(e))
+
+    def select_folder(self, foldername: str) -> tuple:
+        assert not self._idling
+        return self.conn.folder.set(foldername)
+
+    def select_config_folder(self, config_name: str):
+        """Return info about selected folder if it is
+        configured, otherwise None.
+        """
+        if "_" not in config_name:
+            config_name = f"configured_{config_name}_folder"
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+        return None
+
+    def list_folders(self) -> list[str]:
+        """return list of all existing folder names."""
+        assert not self._idling
+        return [folder.name for folder in self.conn.folder.list()]
+
+    def delete(self, uid_list: str, expunge=True):
+        """delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.client.uid("STORE", uid_list, "+FLAGS", r"(\Deleted)")
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self) -> list[MailMessage]:
+        assert not self._idling
+        return list(self.conn.fetch())
+
+    def get_unread_messages(self) -> list[str]:
+        assert not self._idling
+        return [msg.uid for msg in self.conn.fetch(AND(seen=False))]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.flag(messages, MailMessageFlags.SEEN, True)
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self) -> int:
+        return len(self.get_unread_messages())
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            for msg in self.conn.fetch(mark_seen=False):
+                body = getattr(msg.obj, "text", None)
+                if not body:
+                    body = getattr(msg.obj, "html", None)
+                if not body:
+                    log("Message", msg.uid, "has empty body")
+                    continue
+
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(msg.uid))
+                fn.write_bytes(body)
+                log("Message", msg.uid, fn)
+                log(
+                    "Message",
+                    msg.uid,
+                    msg.flags,
+                    "Message-Id:",
+                    msg.obj.get("Message-Id"),
+                )
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    @contextmanager
+    def idle(self):
+        """return Idle ContextManager."""
+        idle_manager = IdleManager(self)
+        try:
+            yield idle_manager
+        finally:
+            idle_manager.done()
+
+    def append(self, folder: str, msg: str):
+        """Upload a message to *folder*.
+        Trailing whitespace or a linebreak at the beginning will be removed automatically.
+        """
+        if msg.startswith("\n"):
+            msg = msg[1:]
+        msg = "\n".join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(bytes(msg, encoding="ascii"), folder)
+
+    def get_uid_by_message_id(self, message_id) -> str:
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header("MESSAGE-ID", message_id)))]
+        if len(msgs) == 0:
+            raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
+        return msgs[0]
+
+
+class IdleManager:
+    def __init__(self, direct_imap) -> None:
+        self.direct_imap = direct_imap
+        self.log = direct_imap.account.log
+        # fetch latest messages before starting idle so that it only
+        # returns messages that arrive anew
+        self.direct_imap.conn.fetch("1:*")
+        self.direct_imap.conn.idle.start()
+
+    def check(self, timeout=None) -> list[bytes]:
+        """(blocking) wait for next idle message from server."""
+        self.log("imap-direct: calling idle_check")
+        res = self.direct_imap.conn.idle.poll(timeout=timeout)
+        self.log(f"imap-direct: idle_check returned {res!r}")
+        return res
+
+    def wait_for_new_message(self, timeout=None) -> bytes:
+        while True:
+            for item in self.check(timeout=timeout):
+                if b"EXISTS" in item or b"RECENT" in item:
+                    return item
+
+    def wait_for_seen(self, timeout=None) -> int:
+        """Return first message with SEEN flag from a running idle-stream."""
+        while True:
+            for item in self.check(timeout=timeout):
+                if FETCH in item:
+                    self.log(str(item))
+                    if FLAGS in item and rb"\Seen" in item:
+                        return int(item.split(b" ")[1])
+
+    def done(self):
+        """send idle-done to server if we are currently in idle mode."""
+        return self.direct_imap.conn.idle.stop()

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -1,7 +1,10 @@
 """High-level classes for event processing and filtering."""
+
+from __future__ import annotations
+
 import re
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union
+from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Union
 
 from .const import EventType
 
@@ -262,9 +265,9 @@ class HookCollection:
     """
 
     def __init__(self) -> None:
-        self._hooks: Set[Tuple[Callable, Union[type, EventFilter]]] = set()
+        self._hooks: set[tuple[Callable, Union[type, EventFilter]]] = set()
 
-    def __iter__(self) -> Iterator[Tuple[Callable, Union[type, EventFilter]]]:
+    def __iter__(self) -> Iterator[tuple[Callable, Union[type, EventFilter]]]:
         return iter(self._hooks)
 
     def on(self, event: Union[type, EventFilter]) -> Callable:  # noqa

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -2,7 +2,8 @@ import json
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Optional, Union
 
-from ._utils import AttrDict
+from ._utils import AttrDict, futuremethod
+from .const import EventType
 from .contact import Contact
 
 if TYPE_CHECKING:
@@ -21,9 +22,10 @@ class Message:
     def _rpc(self) -> "Rpc":
         return self.account._rpc
 
-    def send_reaction(self, *reaction: str):
+    def send_reaction(self, *reaction: str) -> "Message":
         """Send a reaction to this message."""
-        self._rpc.send_reaction(self.account.id, self.id, reaction)
+        msg_id = self._rpc.send_reaction(self.account.id, self.id, reaction)
+        return Message(self.account, msg_id)
 
     def get_snapshot(self) -> AttrDict:
         """Get a snapshot with the properties of this message."""
@@ -61,3 +63,18 @@ class Message:
 
     def get_webxdc_info(self) -> dict:
         return self._rpc.get_webxdc_info(self.account.id, self.id)
+
+    def wait_until_delivered(self) -> None:
+        """Consume events until the message is delivered."""
+        while True:
+            event = self.account.wait_for_event()
+            if event.kind == EventType.MSG_DELIVERED and event.msg_id == self.id:
+                break
+
+    @futuremethod
+    def send_webxdc_realtime_advertisement(self):
+        yield self._rpc.send_webxdc_realtime_advertisement.future(self.account.id, self.id)
+
+    @futuremethod
+    def send_webxdc_realtime_data(self, data) -> None:
+        yield self._rpc.send_webxdc_realtime_data.future(self.account.id, self.id, list(data))

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,10 +1,13 @@
+from __future__ import annotations
+
 import os
 import random
-from typing import AsyncGenerator, List, Optional
+from typing import AsyncGenerator, Optional
 
 import pytest
 
-from . import Account, AttrDict, Bot, Client, DeltaChat, EventType, Message
+from . import Account, AttrDict, Bot, Chat, Client, DeltaChat, EventType, Message
+from ._utils import futuremethod
 from .rpc import Rpc
 
 
@@ -37,9 +40,10 @@ class ACFactory:
         assert not account.is_configured()
         return account
 
-    def new_configured_account(self) -> Account:
+    @futuremethod
+    def new_configured_account(self):
         account = self.new_preconfigured_account()
-        account.configure()
+        yield account.configure.future()
         assert account.is_configured()
         return account
 
@@ -49,17 +53,15 @@ class ACFactory:
         bot.configure(credentials["email"], credentials["password"])
         return bot
 
-    def get_online_account(self) -> Account:
-        account = self.new_configured_account()
-        account.start_io()
-        while True:
-            event = account.wait_for_event()
-            if event.kind == EventType.IMAP_INBOX_IDLE:
-                break
+    @futuremethod
+    def get_online_account(self):
+        account = yield self.new_configured_account.future()
+        account.bring_online()
         return account
 
-    def get_online_accounts(self, num: int) -> List[Account]:
-        return [self.get_online_account() for _ in range(num)]
+    def get_online_accounts(self, num: int) -> list[Account]:
+        futures = [self.get_online_account.future() for _ in range(num)]
+        return [f() for f in futures]
 
     def resetup_account(self, ac: Account) -> Account:
         """Resetup account from scratch, losing the encryption key."""
@@ -71,6 +73,10 @@ class ACFactory:
         ac_clone.configure()
         return ac_clone
 
+    def get_accepted_chat(self, ac1: Account, ac2: Account) -> Chat:
+        ac2.create_chat(ac1)
+        return ac1.create_chat(ac2)
+
     def send_message(
         self,
         to_account: Account,
@@ -108,13 +114,13 @@ class ACFactory:
         return to_client.run_until(lambda e: e.kind == EventType.INCOMING_MSG)
 
 
-@pytest.fixture()
+@pytest.fixture
 def rpc(tmp_path) -> AsyncGenerator:
     rpc_server = Rpc(accounts_dir=str(tmp_path / "accounts"))
     with rpc_server:
         yield rpc_server
 
 
-@pytest.fixture()
+@pytest.fixture
 def acfactory(rpc) -> AsyncGenerator:
     return ACFactory(DeltaChat(rpc))

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,18 +1,62 @@
+from __future__ import annotations
+
 import itertools
 import json
 import logging
 import os
 import subprocess
 import sys
-from queue import Queue
+from queue import Empty, Queue
 from threading import Event, Thread
-from typing import Any, Dict, Iterator, Optional
+from typing import Any, Iterator, Optional
 
 
 class JsonRpcError(Exception):
     pass
 
 
+class RpcFuture:
+    def __init__(self, rpc: "Rpc", request_id: int, event: Event):
+        self.rpc = rpc
+        self.request_id = request_id
+        self.event = event
+
+    def __call__(self):
+        self.event.wait()
+        response = self.rpc.request_results.pop(self.request_id)
+        if "error" in response:
+            raise JsonRpcError(response["error"])
+        if "result" in response:
+            return response["result"]
+        return None
+
+
+class RpcMethod:
+    def __init__(self, rpc: "Rpc", name: str):
+        self.rpc = rpc
+        self.name = name
+
+    def __call__(self, *args) -> Any:
+        """Synchronously calls JSON-RPC method."""
+        future = self.future(*args)
+        return future()
+
+    def future(self, *args) -> Any:
+        """Asynchronously calls JSON-RPC method."""
+        request_id = next(self.rpc.id_iterator)
+        request = {
+            "jsonrpc": "2.0",
+            "method": self.name,
+            "params": args,
+            "id": request_id,
+        }
+        event = Event()
+        self.rpc.request_events[request_id] = event
+        self.rpc.request_queue.put(request)
+
+        return RpcFuture(self.rpc, request_id, event)
+
+
 class Rpc:
     def __init__(self, accounts_dir: Optional[str] = None, **kwargs):
         """The given arguments will be passed to subprocess.Popen()"""
@@ -25,11 +69,11 @@ class Rpc:
         self._kwargs = kwargs
         self.process: subprocess.Popen
         self.id_iterator: Iterator[int]
-        self.event_queues: Dict[int, Queue]
+        self.event_queues: dict[int, Queue]
         # Map from request ID to `threading.Event`.
-        self.request_events: Dict[int, Event]
+        self.request_events: dict[int, Event]
         # Map from request ID to the result.
-        self.request_results: Dict[int, Any]
+        self.request_results: dict[int, Any]
         self.request_queue: Queue[Any]
         self.closing: bool
         self.reader_thread: Thread
@@ -144,25 +188,14 @@ class Rpc:
         queue = self.get_queue(account_id)
         return queue.get()
 
+    def clear_all_events(self, account_id: int):
+        """Removes all queued-up events for a given account. Useful for tests."""
+        queue = self.get_queue(account_id)
+        try:
+            while True:
+                queue.get_nowait()
+        except Empty:
+            pass
+
     def __getattr__(self, attr: str):
-        def method(*args) -> Any:
-            request_id = next(self.id_iterator)
-            request = {
-                "jsonrpc": "2.0",
-                "method": attr,
-                "params": args,
-                "id": request_id,
-            }
-            event = Event()
-            self.request_events[request_id] = event
-            self.request_queue.put(request)
-            event.wait()
-
-            response = self.request_results.pop(request_id)
-            if "error" in response:
-                raise JsonRpcError(response["error"])
-            if "result" in response:
-                return response["result"]
-            return None
-
-        return method
+        return RpcMethod(self, attr)

deltachat-rpc-client/tests/test_chatlist_events.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import base64
+import os
+from typing import TYPE_CHECKING
+
+from deltachat_rpc_client import Account, EventType, const
+
+if TYPE_CHECKING:
+    from deltachat_rpc_client.pytestplugin import ACFactory
+
+
+def wait_for_chatlist_and_specific_item(account, chat_id):
+    first_event = ""
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.CHATLIST_CHANGED:
+            first_event = "change"
+            break
+        if event.kind == EventType.CHATLIST_ITEM_CHANGED and event.chat_id == chat_id:
+            first_event = "item_change"
+            break
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.CHATLIST_CHANGED and first_event == "item_change":
+            break
+        if event.kind == EventType.CHATLIST_ITEM_CHANGED and event.chat_id == chat_id and first_event == "change":
+            break
+
+
+def wait_for_chatlist_specific_item(account, chat_id):
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.CHATLIST_ITEM_CHANGED and event.chat_id == chat_id:
+            break
+
+
+def wait_for_chatlist(account):
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.CHATLIST_CHANGED:
+            break
+
+
+def test_delivery_status(acfactory: ACFactory) -> None:
+    """
+    Test change status on chatlistitem when status changes (delivered, read)
+    """
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    alice.clear_all_events()
+    bob.stop_io()
+    alice.stop_io()
+    alice_chat_bob.send_text("hi")
+    wait_for_chatlist_and_specific_item(alice, chat_id=alice_chat_bob.id)
+
+    alice.clear_all_events()
+    alice.start_io()
+    wait_for_chatlist_specific_item(alice, chat_id=alice_chat_bob.id)
+
+    bob.clear_all_events()
+    bob.start_io()
+
+    event = bob.wait_for_incoming_msg_event()
+    msg = bob.get_message_by_id(event.msg_id)
+    msg.get_snapshot().chat.accept()
+    msg.mark_seen()
+
+    chat_item = alice._rpc.get_chatlist_items_by_entries(alice.id, [alice_chat_bob.id])[str(alice_chat_bob.id)]
+    assert chat_item["summaryStatus"] == const.MessageState.OUT_DELIVERED
+
+    alice.clear_all_events()
+
+    while True:
+        event = alice.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    wait_for_chatlist_specific_item(alice, chat_id=alice_chat_bob.id)
+    chat_item = alice._rpc.get_chatlist_items_by_entries(alice.id, [alice_chat_bob.id])[str(alice_chat_bob.id)]
+    assert chat_item["summaryStatus"] == const.MessageState.OUT_MDN_RCVD
+
+
+def test_delivery_status_failed(acfactory: ACFactory) -> None:
+    """
+    Test change status on chatlistitem when status changes failed
+    """
+    (alice,) = acfactory.get_online_accounts(1)
+
+    invalid_contact = alice.create_contact("example@example.com", "invalid address")
+    invalid_chat = alice.get_chat_by_id(alice._rpc.create_chat_by_contact_id(alice.id, invalid_contact.id))
+
+    alice.clear_all_events()
+
+    failing_message = invalid_chat.send_text("test")
+
+    wait_for_chatlist_and_specific_item(alice, invalid_chat.id)
+
+    assert failing_message.get_snapshot().state == const.MessageState.OUT_PENDING
+
+    while True:
+        event = alice.wait_for_event()
+        if event.kind == EventType.MSG_FAILED:
+            break
+
+    wait_for_chatlist_specific_item(alice, invalid_chat.id)
+
+    assert failing_message.get_snapshot().state == const.MessageState.OUT_FAILED
+
+
+def test_download_on_demand(acfactory: ACFactory) -> None:
+    """
+    Test if download on demand emits chatlist update events.
+    This is only needed for last message in chat, but finding that out is too expensive, so it's always emitted
+    """
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("hi")
+
+    alice.set_config("download_limit", "1")
+
+    msg = bob.wait_for_incoming_msg()
+    chat_id = msg.get_snapshot().chat_id
+    msg.get_snapshot().chat.accept()
+    bob.get_chat_by_id(chat_id).send_message(
+        "Hello World, this message is bigger than 5 bytes",
+        html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
+    )
+
+    message = alice.wait_for_incoming_msg()
+    snapshot = message.get_snapshot()
+    assert snapshot.download_state == const.DownloadState.AVAILABLE
+
+    alice.clear_all_events()
+
+    snapshot = message.get_snapshot()
+    chat_id = snapshot.chat_id
+    alice._rpc.download_full_message(alice.id, message.id)
+
+    wait_for_chatlist_specific_item(alice, chat_id)
+
+
+def get_multi_account_test_setup(acfactory: ACFactory) -> [Account, Account, Account]:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("hi")
+
+    bob.wait_for_incoming_msg_event()
+
+    alice_second_device: Account = acfactory.get_unconfigured_account()
+
+    alice._rpc.provide_backup.future(alice.id)
+    backup_code = alice._rpc.get_backup_qr(alice.id)
+    alice_second_device._rpc.get_backup(alice_second_device.id, backup_code)
+    alice_second_device.start_io()
+    alice.clear_all_events()
+    alice_second_device.clear_all_events()
+    bob.clear_all_events()
+    return [alice, alice_second_device, bob, alice_chat_bob]
+
+
+def test_imap_sync_seen_msgs(acfactory: ACFactory) -> None:
+    """
+    Test that chatlist changed events are emitted for the second device
+    when the message is marked as read on the first device
+    """
+    alice, alice_second_device, bob, alice_chat_bob = get_multi_account_test_setup(acfactory)
+
+    alice_chat_bob.send_text("hello")
+
+    msg = bob.wait_for_incoming_msg()
+    bob_chat_id = msg.get_snapshot().chat_id
+    msg.get_snapshot().chat.accept()
+
+    alice.clear_all_events()
+    alice_second_device.clear_all_events()
+    bob.get_chat_by_id(bob_chat_id).send_text("hello")
+
+    # make sure alice_second_device already received the message
+    alice_second_device.wait_for_incoming_msg_event()
+
+    msg = alice.wait_for_incoming_msg()
+    alice_second_device.clear_all_events()
+    msg.mark_seen()
+
+    wait_for_chatlist_specific_item(bob, bob_chat_id)
+    wait_for_chatlist_specific_item(alice, alice_chat_bob.id)
+
+
+def test_multidevice_sync_chat(acfactory: ACFactory) -> None:
+    """
+    Test multidevice sync: syncing chat visibility and muting across multiple devices
+    """
+    alice, alice_second_device, bob, alice_chat_bob = get_multi_account_test_setup(acfactory)
+
+    alice_chat_bob.archive()
+    wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
+    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().archived
+
+    alice_second_device.clear_all_events()
+    alice_chat_bob.pin()
+    wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
+    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().pinned
+
+    alice_second_device.clear_all_events()
+    alice_chat_bob.mute()
+    wait_for_chatlist_specific_item(alice_second_device, alice_chat_bob.id)
+    assert alice_second_device.get_chat_by_id(alice_chat_bob.id).get_basic_snapshot().is_muted

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""
+Testing webxdc iroh connectivity
+
+If you want to debug iroh at rust-trace/log level set
+
+    RUST_LOG=iroh_net=trace,iroh_gossip=trace
+"""
+
+import sys
+import threading
+import time
+
+import pytest
+
+from deltachat_rpc_client import EventType
+
+
+@pytest.fixture
+def path_to_webxdc(request):
+    p = request.path.parent.parent.parent.joinpath("test-data/webxdc/chess.xdc")
+    assert p.exists()
+    return str(p)
+
+
+def log(msg):
+    print()
+    print("*" * 80 + "\n" + msg + "\n", file=sys.stderr)
+    print()
+
+
+def setup_realtime_webxdc(ac1, ac2, path_to_webxdc):
+    assert ac1.get_config("webxdc_realtime_enabled") == "1"
+    assert ac2.get_config("webxdc_realtime_enabled") == "1"
+    ac1_ac2_chat = ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    assert ac2_webxdc_msg.get_snapshot().text == "play"
+
+    # send iroh announcements simultaneously
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    return ac1_webxdc_msg, ac2_webxdc_msg
+
+
+def setup_thread_send_realtime_data(msg, data):
+    def thread_run():
+        for _i in range(10):
+            msg.send_webxdc_realtime_data(data)
+            time.sleep(1)
+
+    threading.Thread(target=thread_run, daemon=True).start()
+
+
+def wait_receive_realtime_data(msg_data_list):
+    account = msg_data_list[0][0].account
+    msg_data_list = msg_data_list[:]
+
+    log(f"account {account.id}: waiting for realtime data {msg_data_list}")
+    while msg_data_list:
+        event = account.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            for i, (msg, data) in enumerate(msg_data_list):
+                if msg.id == event.msg_id:
+                    assert list(data) == event.data
+                    log(f"msg {msg.id}: got correct realtime data {data}")
+                    del msg_data_list[i]
+                    break
+
+
+def test_realtime_sequentially(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection sequentially."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+    ac1.create_chat(ac2)
+    ac2.create_chat(ac1)
+
+    # share a webxdc app between ac1 and ac2
+    ac1_webxdc_msg = acfactory.send_message(from_account=ac1, to_account=ac2, text="play", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    snapshot = ac2_webxdc_msg.get_snapshot()
+    assert snapshot.text == "play"
+
+    # send iroh announcements sequentially
+    log("sending ac1 -> ac2 realtime advertisement and additional message")
+    ac1_webxdc_msg.send_webxdc_realtime_advertisement()
+    acfactory.send_message(from_account=ac1, to_account=ac2, text="ping1")
+
+    log("waiting for incoming message on ac2")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping1"
+
+    log("sending ac2 -> ac1 realtime advertisement and additional message")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+    acfactory.send_message(from_account=ac2, to_account=ac1, text="ping2")
+
+    log("waiting for incoming message on ac1")
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "ping2"
+
+    log("sending realtime data ac1 -> ac2")
+    ac1_webxdc_msg.send_webxdc_realtime_data(b"foo")
+
+    log("ac2: waiting for realtime data")
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert event.data == list(b"foo")
+            break
+
+
+def test_realtime_simultaneously(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection simultaneously."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
+    wait_receive_realtime_data([(ac2_webxdc_msg, [10])])
+
+
+def test_two_parallel_realtime_simultaneously(acfactory, path_to_webxdc):
+    """Test two peers trying to establish connection simultaneously."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+    ac1_webxdc_msg2, ac2_webxdc_msg2 = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, [10])
+    setup_thread_send_realtime_data(ac1_webxdc_msg2, [20])
+    setup_thread_send_realtime_data(ac2_webxdc_msg, [30])
+    setup_thread_send_realtime_data(ac2_webxdc_msg2, [40])
+
+    wait_receive_realtime_data([(ac1_webxdc_msg, [30]), (ac1_webxdc_msg2, [40])])
+    wait_receive_realtime_data([(ac2_webxdc_msg, [10]), (ac2_webxdc_msg2, [20])])
+
+
+def test_no_duplicate_messages(acfactory, path_to_webxdc):
+    """Test that messages are received only once."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_ac2_chat = ac1.create_chat(ac2)
+
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="webxdc", file=path_to_webxdc)
+
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    ac2_webxdc_msg.get_snapshot().chat.accept()
+    assert ac2_webxdc_msg.get_snapshot().text == "webxdc"
+
+    # Issue a "send" call in parallel with sending advertisement.
+    # Previously due to a bug this caused subscribing to the channel twice.
+    ac2_webxdc_msg.send_webxdc_realtime_data.future(b"foobar")
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+
+    def thread_run():
+        for i in range(10):
+            data = str(i).encode()
+            ac1_webxdc_msg.send_webxdc_realtime_data(data)
+            time.sleep(1)
+
+    threading.Thread(target=thread_run, daemon=True).start()
+
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            n = int(bytes(event.data).decode())
+            break
+
+    while 1:
+        event = ac2.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_DATA:
+            assert int(bytes(event.data).decode()) > n
+            break
+
+
+def test_no_reordering(acfactory, path_to_webxdc):
+    """Test that sending a lot of realtime messages does not result in reordering."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_webxdc_msg, ac2_webxdc_msg = setup_realtime_webxdc(ac1, ac2, path_to_webxdc)
+
+    setup_thread_send_realtime_data(ac1_webxdc_msg, b"hello")
+    wait_receive_realtime_data([(ac2_webxdc_msg, b"hello")])
+
+    for i in range(200):
+        ac1_webxdc_msg.send_webxdc_realtime_data([i])
+
+    for i in range(200):
+        while 1:
+            event = ac2.wait_for_event()
+            if event.kind == EventType.WEBXDC_REALTIME_DATA and bytes(event.data) != b"hello":
+                if event.data[0] == i:
+                    break
+                pytest.fail("Reordering detected")

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,12 +1,14 @@
 import logging
 
-from deltachat_rpc_client import Chat, SpecialContactId
+import pytest
+
+from deltachat_rpc_client import Chat, EventType, SpecialContactId
 
 
-def test_qr_setup_contact(acfactory) -> None:
+def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
 
     alice.wait_for_securejoin_inviter_success()
@@ -23,16 +25,51 @@ def test_qr_setup_contact(acfactory) -> None:
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
+    # Test that if Bob changes the key, backwards verification is lost.
+    logging.info("Bob 2 is created")
+    bob2 = acfactory.new_configured_account()
+    bob2.export_self_keys(tmp_path)
 
-def test_qr_securejoin(acfactory):
+    logging.info("Bob imports a key")
+    bob.import_self_keys(tmp_path)
+
+    assert bob.get_config("key_id") == "2"
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert not bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_setup_contact_svg(acfactory) -> None:
+    alice = acfactory.new_configured_account()
+    _, _, domain = alice.get_config("addr").rpartition("@")
+
+    _qr_code, svg = alice.get_qr_code_svg()
+
+    # Test that email address is in SVG
+    # when we have no display name.
+    # Check only the domain name, because
+    # long address may be split over multiple lines
+    # and not matched.
+    assert domain in svg
+
+    alice.set_config("displayname", "Alice")
+
+    # Test that display name is used
+    # in SVG and no address is visible.
+    _qr_code, svg = alice.get_qr_code_svg()
+    assert domain not in svg
+    assert "Alice" in svg
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect):
     alice, bob = acfactory.get_online_accounts(2)
 
     logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=True)
-    assert alice_chat.get_basic_snapshot().is_protected
+    alice_chat = alice.create_group("Verified group", protect=protect)
+    assert alice_chat.get_basic_snapshot().is_protected == protect
 
     logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
     # Check that at least some of the handshake messages are deleted.
@@ -53,7 +90,7 @@ def test_qr_securejoin(acfactory):
 
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
-    assert snapshot.chat.get_basic_snapshot().is_protected
+    assert snapshot.chat.get_basic_snapshot().is_protected == protect
 
     # Test that Bob verified Alice's profile.
     bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
@@ -77,7 +114,7 @@ def test_qr_securejoin_contact_request(acfactory) -> None:
 
     alice_chat = alice.create_group("Verified group", protect=True)
     logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
+    qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
     while True:
         event = bob.wait_for_event()
@@ -92,7 +129,7 @@ def test_qr_readreceipt(acfactory) -> None:
     alice, bob, charlie = acfactory.get_online_accounts(3)
 
     logging.info("Bob and Charlie setup contact with Alice")
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
 
     bob.secure_join(qr_code)
     charlie.secure_join(qr_code)
@@ -150,6 +187,21 @@ def test_qr_readreceipt(acfactory) -> None:
     assert not bob.get_chat_by_contact(bob_contact_charlie)
 
 
+def test_setup_contact_resetup(acfactory) -> None:
+    """Tests that setup contact works after Alice resets the device and changes the key."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    alice = acfactory.resetup_account(alice)
+
+    qr_code = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+
 def test_verified_group_recovery(acfactory) -> None:
     """Tests verified group recovery by reverifying a member and sending a message in a group."""
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)
@@ -159,9 +211,9 @@ def test_verified_group_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -169,16 +221,16 @@ def test_verified_group_recovery(acfactory) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
@@ -223,9 +275,9 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     assert chat.get_basic_snapshot().is_protected
 
     logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -233,16 +285,16 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
 
     logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
+    qr_code = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
@@ -307,7 +359,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
     ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
 
     logging.info("ac3: verify with ac2")
-    qr_code, _svg = ac2.get_qr_code()
+    qr_code = ac2.get_qr_code()
     ac3.secure_join(qr_code)
     ac2.wait_for_securejoin_inviter_success()
 
@@ -317,7 +369,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
 
     logging.info("ac1: create verified group that ac2 fully joins")
     ch1 = ac1.create_group("Group", protect=True)
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
 
@@ -330,7 +382,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    qr_code, _svg = ch1.get_qr_code()
+    qr_code = ch1.get_qr_code()
     ac2.secure_join(qr_code)
     ac1.remove()
     logging.info("ac2 now has pending bobstate but ac1 is shutoff")
@@ -352,7 +404,7 @@ def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
             break
 
     logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    qr_code, _svg = vg.get_qr_code()
+    qr_code = vg.get_qr_code()
     ac4.secure_join(qr_code)
     ac3.wait_for_securejoin_inviter_success()
     while 1:
@@ -373,7 +425,7 @@ def test_qr_new_group_unblocked(acfactory):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_chat = ac1.create_group("Group for joining", protect=True)
-    qr_code, _svg = ac1_chat.get_qr_code()
+    qr_code = ac1_chat.get_qr_code()
     ac2.secure_join(qr_code)
 
     ac1.wait_for_securejoin_inviter_success()
@@ -396,7 +448,7 @@ def test_aeap_flow_verified(acfactory):
     logging.info("ac1: create verified-group QR, ac2 scans and joins")
     chat = ac1.create_group("hello", protect=True)
     assert chat.get_basic_snapshot().is_protected
-    qr_code, _svg = chat.get_qr_code()
+    qr_code = chat.get_qr_code()
     logging.info("ac2: start QR-code based join-group protocol")
     ac2.secure_join(qr_code)
     ac1.wait_for_securejoin_inviter_success()
@@ -435,12 +487,12 @@ def test_gossip_verification(acfactory) -> None:
     alice, bob, carol = acfactory.get_online_accounts(3)
 
     # Bob verifies Alice.
-    qr_code, _svg = alice.get_qr_code()
+    qr_code = alice.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
     # Bob verifies Carol.
-    qr_code, _svg = carol.get_qr_code()
+    qr_code = carol.get_qr_code()
     bob.secure_join(qr_code)
     bob.wait_for_securejoin_joiner_success()
 
@@ -491,16 +543,16 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
     ac3_chat = ac3.create_group("Verified group", protect=True)
 
     # ac1 joins ac3 group.
-    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac3_qr_code = ac3_chat.get_qr_code()
     ac1.secure_join(ac3_qr_code)
     ac1.wait_for_securejoin_joiner_success()
 
     # ac1 waits for member added message and creates a QR code.
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+    ac1_qr_code = snapshot.chat.get_qr_code()
 
     # ac2 verifies ac1
-    qr_code, _svg = ac1.get_qr_code()
+    qr_code = ac1.get_qr_code()
     ac2.secure_join(qr_code)
     ac2.wait_for_securejoin_joiner_success()
 
@@ -550,3 +602,40 @@ def test_securejoin_after_contact_resetup(acfactory) -> None:
 
     # ac1 is still "not verified" for ac2 due to inconsistent state.
     assert not ac2_contact_ac1.get_snapshot().is_verified
+
+
+def test_withdraw_securejoin_qr(acfactory):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=True)
+    assert alice_chat.get_basic_snapshot().is_protected
+    logging.info("Bob joins verified group")
+
+    qr_code = alice_chat.get_qr_code()
+    bob_chat = bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
+    assert snapshot.chat.get_basic_snapshot().is_protected
+    bob_chat.leave()
+
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Group left by {}.".format(bob.get_config("addr"))
+
+    logging.info("Alice withdraws QR code.")
+    qr = alice.check_qr(qr_code)
+    assert qr["kind"] == "withdrawVerifyGroup"
+    alice.set_config_from_qr(qr_code)
+
+    logging.info("Bob scans withdrawn QR code.")
+    bob_chat = bob.secure_join(qr_code)
+
+    logging.info("Bob scanned withdrawn QR code")
+    while True:
+        event = alice.wait_for_event()
+        if event.kind == EventType.MSGS_CHANGED and event.chat_id != 0:
+            break
+    snapshot = alice.get_message_by_id(event.msg_id).get_snapshot()
+    assert snapshot.text == "Cannot establish guaranteed end-to-end encryption with {}".format(bob.get_config("addr"))

deltachat-rpc-client/tests/test_something.py

@@ -1,10 +1,18 @@
+import base64
 import concurrent.futures
 import json
+import logging
+import os
+import socket
 import subprocess
+import time
 from unittest.mock import MagicMock
 
 import pytest
-from deltachat_rpc_client import EventType, events
+
+from deltachat_rpc_client import Contact, EventType, Message, events
+from deltachat_rpc_client.const import DownloadState, MessageState
+from deltachat_rpc_client.direct_imap import DirectImap
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -63,6 +71,18 @@ def test_configure_starttls(acfactory) -> None:
     assert account.is_configured()
 
 
+def test_configure_ip(acfactory) -> None:
+    account = acfactory.new_preconfigured_account()
+
+    domain = account.get_config("addr").rsplit("@")[-1]
+    ip_address = socket.gethostbyname(domain)
+
+    # This should fail TLS check.
+    account.set_config("mail_server", ip_address)
+    with pytest.raises(JsonRpcError):
+        account.configure()
+
+
 def test_account(acfactory) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
@@ -98,12 +118,12 @@ def test_account(acfactory) -> None:
     assert alice.get_chatlist(snapshot=True)
     assert alice.get_qr_code()
     assert alice.get_fresh_messages()
-    assert alice.get_next_messages()
 
     # Test sending empty message.
     assert len(bob.wait_next_messages()) == 0
     alice_chat_bob.send_text("")
     messages = bob.wait_next_messages()
+    assert bob.get_next_messages() == messages
     assert len(messages) == 1
     message = messages[0]
     snapshot = message.get_snapshot()
@@ -140,12 +160,9 @@ def test_chat(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
     assert snapshot.chat_id == chat_id
@@ -224,12 +241,9 @@ def test_message(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
 
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
@@ -331,7 +345,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -341,7 +355,7 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
@@ -352,6 +366,31 @@ def test_import_export(acfactory, tmp_path) -> None:
     assert alice2.manager.get_system_info()
 
 
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
 def test_openrpc_command_line() -> None:
     """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
     out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
@@ -377,3 +416,222 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
+
+
+def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+
+    # Bob creates chat manually so chat with Alice is accepted.
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    # Alice sends a message to Bob.
+    alice_chat_bob.send_text("Hello Bob!")
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+
+    # Bob sends a message to Alice.
+    bob_chat_alice = snapshot.chat
+    bob_chat_alice.accept()
+    bob_chat_alice.send_text("Hello Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+    # Alice reads Bob's message.
+    message.mark_seen()
+    while True:
+        event = bob.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    # Bob sends a message to Alice, it should also be encrypted.
+    bob_chat_alice.send_text("Hi Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+
+def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
+    """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded
+    messages are received out of order".
+
+    If the Inbox contains X small messages followed by Y large messages followed by Z small
+    messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages.
+
+    This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced
+    with online test as follows:
+    - Bob enables download limit and goes offline.
+    - Alice sends a large message to Bob and reacts to this message with a thumbs-up.
+    - Bob goes online
+    - Bob first processes a reaction message and throws it away because there is no corresponding
+      message, then processes a partially downloaded message.
+    - As a result, Bob does not see a reaction
+    """
+    download_limit = 300000
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1_addr = ac1.get_config("addr")
+    chat = ac1.create_chat(ac2)
+    ac2.set_config("download_limit", str(download_limit))
+    ac2.stop_io()
+
+    logging.info("sending small+large messages from ac1 to ac2")
+    msgs = []
+    msgs.append(chat.send_text("hi"))
+    path = tmp_path / "large"
+    path.write_bytes(os.urandom(download_limit + 1))
+    msgs.append(chat.send_file(str(path)))
+    for m in msgs:
+        m.wait_until_delivered()
+
+    logging.info("sending a reaction to the large message from ac1 to ac2")
+    # TODO: Find the reason of an occasional message reordering on the server (so that the reaction
+    # has a lower UID than the previous message). W/a is to sleep for some time to let the reaction
+    # have a later INTERNALDATE.
+    time.sleep(1.1)
+    react_str = "\N{THUMBS UP SIGN}"
+    msgs.append(msgs[-1].send_reaction(react_str))
+    msgs[-1].wait_until_delivered()
+
+    ac2.start_io()
+
+    logging.info("wait for ac2 to receive a reaction")
+    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
+    assert msg2.get_sender_contact().get_snapshot().address == ac1_addr
+    assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE
+    reactions = msg2.get_reactions()
+    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
+    assert len(contacts) == 1
+    assert contacts[0].get_snapshot().address == ac1_addr
+    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
+
+
+def test_reactions_for_a_reordering_move(acfactory):
+    """When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
+    their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
+    processed by receive_imf in the wrong order, and, particularly, reactions were processed before
+    messages they refer to and thus dropped.
+    """
+    (ac1,) = acfactory.get_online_accounts(1)
+    ac2 = acfactory.new_preconfigured_account()
+    ac2.configure()
+    ac2.set_config("mvbox_move", "1")
+    ac2.bring_online()
+    chat1 = acfactory.get_accepted_chat(ac1, ac2)
+    ac2.stop_io()
+
+    logging.info("sending message + reaction from ac1 to ac2")
+    msg1 = chat1.send_text("hi")
+    msg1.wait_until_delivered()
+    # It's is sad, but messages must differ in their INTERNALDATEs to be processed in the correct
+    # order by DC, and most (if not all) mail servers provide only seconds precision.
+    time.sleep(1.1)
+    react_str = "\N{THUMBS UP SIGN}"
+    msg1.send_reaction(react_str).wait_until_delivered()
+
+    logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
+    ac2_direct_imap = DirectImap(ac2)
+    ac2_direct_imap.connect()
+    for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
+        ac2_direct_imap.conn.move(uid, "DeltaChat")
+
+    logging.info("receiving messages by ac2")
+    ac2.start_io()
+    msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
+    assert msg2.get_snapshot().text == msg1.get_snapshot().text
+    reactions = msg2.get_reactions()
+    contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
+    assert len(contacts) == 1
+    assert contacts[0].get_snapshot().address == ac1.get_config("addr")
+    assert list(reactions.reactions_by_contact.values())[0] == [react_str]
+
+
+@pytest.mark.parametrize("n_accounts", [3, 2])
+def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
+    download_limit = 300000
+
+    alice, *others = acfactory.get_online_accounts(n_accounts)
+    bob = others[0]
+
+    alice_group = alice.create_group("test group")
+    for account in others:
+        chat = account.create_chat(alice)
+        chat.send_text("Hello Alice!")
+        assert alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot().text == "Hello Alice!"
+
+        contact_addr = account.get_config("addr")
+        contact = alice.create_contact(contact_addr, "")
+
+        alice_group.add_contact(contact)
+
+    if n_accounts == 2:
+        bob_chat_alice = bob.create_chat(alice)
+    bob.set_config("download_limit", str(download_limit))
+
+    alice_group.send_text("hi")
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "hi"
+    bob_group = snapshot.chat
+
+    path = tmp_path / "large"
+    path.write_bytes(os.urandom(download_limit + 1))
+
+    for i in range(10):
+        logging.info("Sending message %s", i)
+        alice_group.send_file(str(path))
+        snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        assert snapshot.download_state == DownloadState.AVAILABLE
+        if n_accounts > 2:
+            assert snapshot.chat == bob_group
+        else:
+            # Group contains only Alice and Bob,
+            # so partially downloaded messages are
+            # hard to distinguish from private replies to group messages.
+            #
+            # Message may be a private reply, so we assign it to 1:1 chat with Alice.
+            assert snapshot.chat == bob_chat_alice
+
+
+def test_markseen_contact_request(acfactory, tmp_path):
+    """
+    Test that seen status is synchronized for contact request messages
+    even though read receipt is not sent.
+    """
+    alice, bob = acfactory.get_online_accounts(2)
+
+    # Bob sets up a second device.
+    bob.export_backup(tmp_path)
+    files = list(tmp_path.glob("*.tar"))
+    bob2 = acfactory.get_unconfigured_account()
+    bob2.import_backup(files[0])
+    bob2.start_io()
+
+    alice_chat_bob = alice.create_chat(bob)
+    alice_chat_bob.send_text("Hello Bob!")
+
+    message = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id)
+    message2 = bob2.get_message_by_id(bob2.wait_for_incoming_msg_event().msg_id)
+    assert message2.get_snapshot().state == MessageState.IN_FRESH
+
+    message.mark_seen()
+    while True:
+        event = bob2.wait_for_event()
+        if event.kind == EventType.MSGS_NOTICED:
+            break
+    assert message2.get_snapshot().state == MessageState.IN_SEEN
+
+
+def test_get_http_response(acfactory):
+    alice = acfactory.new_configured_account()
+    http_response = alice._rpc.get_http_response(alice.id, "https://example.org")
+    assert http_response["mimetype"] == "text/html"
+    assert b"<title>Example Domain</title>" in base64.b64decode((http_response["blob"] + "==").encode())

deltachat-rpc-client/tests/test_vcard.py

@@ -0,0 +1,15 @@
+def test_vcard(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_contact_charlie = alice.create_contact("charlie@example.org", "Charlie")
+
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_contact(alice_contact_charlie)
+
+    event = bob.wait_for_incoming_msg_event()
+    message = bob.get_message_by_id(event.msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.vcard_contact
+    assert snapshot.vcard_contact.addr == "charlie@example.org"

deltachat-rpc-client/tox.ini

@@ -22,10 +22,9 @@ skipsdist = True
 skip_install = True
 deps =
     ruff
-    black
 commands =
-    black --quiet --check --diff src/ examples/ tests/
-    ruff src/ examples/ tests/
+    ruff format --quiet --diff src/ examples/ tests/
+    ruff check src/ examples/ tests/
 
 [pytest]
 timeout = 300

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.132.0"
+version = "1.142.6"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -10,18 +10,18 @@ keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
 
 [dependencies]
-deltachat-jsonrpc = { path = "../deltachat-jsonrpc", default-features = false }
-deltachat = { path = "..", default-features = false }
+deltachat-jsonrpc = { workspace = true }
+deltachat = { workspace = true }
 
-anyhow = "1"
-env_logger = { version = "0.10.0" }
-futures-lite = "2.0.0"
-log = "0.4"
-serde_json = "1.0.105"
-serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.33.0", features = ["io-std"] }
-tokio-util = "0.7.9"
-yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+anyhow = { workspace = true }
+futures-lite = { workspace = true }
+log = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+tokio = { workspace = true, features = ["io-std"] }
+tokio-util = { workspace = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
 
 [features]
 default = ["vendored"]

deltachat-rpc-server/README.md

@@ -30,7 +30,7 @@ deltachat-rpc-server
 The common use case for this program is to create bindings to use Delta Chat core from programming
 languages other than Rust, for example:
 
-1. Python: https://github.com/deltachat/deltachat-core-rust/tree/master/deltachat-rpc-client/
+1. Python: https://pypi.org/project/deltachat-rpc-client/
 2. Go: https://github.com/deltachat/deltachat-rpc-client-go/
 
 Run `deltachat-rpc-server --version` to check the version of the server.

deltachat-rpc-server/npm-package/.gitignore

@@ -0,0 +1,3 @@
+platform_package
+*.tgz
+package-lock.json

deltachat-rpc-server/npm-package/.npmignore

@@ -0,0 +1,3 @@
+platform_package/*
+scripts/
+*.tgz

deltachat-rpc-server/npm-package/README.md

@@ -0,0 +1,78 @@
+## npm package for deltachat-rpc-server
+
+This is the successor of `deltachat-node`,
+it does not use NAPI bindings but instead uses stdio executables
+to let you talk to core over jsonrpc over stdio.
+This simplifies cross-compilation and even reduces binary size (no CFFI layer and no NAPI layer).
+
+## Usage
+
+> The **minimum** nodejs version for this package is `16`
+
+```
+npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client
+```
+
+```js
+import { startDeltaChat } from "@deltachat/stdio-rpc-server";
+import { C } from "@deltachat/jsonrpc-client";
+
+async function main() {
+    const dc = await startDeltaChat("deltachat-data");
+    console.log(await dc.rpc.getSystemInfo());
+    dc.close()
+}
+main()
+```
+
+For a more complete example refer to https://github.com/deltachat-bot/echo/pull/69/files (TODO change link when pr is merged).
+
+## How to use on an unsupported platform
+
+<!-- todo instructions, will uses an env var for pointing to `deltachat-rpc-server` binary -->
+
+<!-- todo copy parts from https://github.com/deltachat/deltachat-desktop/blob/7045c6f549e4b9d5caa0709d5bd314bbd9fd53db/docs/UPDATE_CORE.md -->
+
+## How does it work when you install it
+
+NPM automatically installs platform dependent optional dependencies when `os` and `cpu` fields are set correctly.
+
+references:
+
+- https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages, [webarchive version](https://web.archive.org/web/20240309234250/https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages)
+- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#cpu
+- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#os
+
+When you import this package it searches for the rpc server in the following locations and order:
+
+1. `DELTA_CHAT_RPC_SERVER` environment variable
+2. use the PATH when `{takeVersionFromPATH: true}` is supplied in the options. 
+3. prebuilds in npm packages
+
+so by default it uses the prebuilds.
+
+## How do you built this package in CI
+
+- To build platform packages, run the `build_platform_package.py` script:
+  ```
+  python3 build_platform_package.py <cargo-target>
+  # example
+  python3 build_platform_package.py x86_64-apple-darwin
+  ```
+- Then pass it as an artifact to the last CI action that publishes the main package.
+- upload all packages from `deltachat-rpc-server/npm-package/platform_package`.
+- then publish `deltachat-rpc-server/npm-package`,
+  - this will run `update_optional_dependencies_and_version.js` (in the `prepack` script),
+    which puts all platform packages into `optionalDependencies` and updates the `version` in `package.json`
+
+## How to build a version you can use localy on your host machine for development
+
+You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have seperate scripts for making it work for local installation.
+
+- If you just need your host platform run `python scripts/make_local_dev_version.py`
+- note: this clears the `platform_package` folder
+- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple plaftorms with `build_platform_package.py`
+
+## Thanks to nlnet
+
+The initial work on this package was funded by nlnet as part of the [Delta Tauri](https://nlnet.nl/project/DeltaTauri/) Project.

deltachat-rpc-server/npm-package/index.d.ts

@@ -0,0 +1,42 @@
+import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
+
+export interface SearchOptions {
+  /** whether take deltachat-rpc-server inside of $PATH*/
+  takeVersionFromPATH: boolean;
+
+  /** whether to disable the DELTA_CHAT_RPC_SERVER environment variable */
+  disableEnvPath: boolean;
+}
+
+/**
+ * 
+ * @returns absolute path to deltachat-rpc-server binary
+ * @throws when it is not found
+ */
+export function getRPCServerPath(
+  options?: Partial<SearchOptions>
+): Promise<string>;
+
+
+
+export type DeltaChatOverJsonRpcServer = StdioDeltaChat & {
+    readonly pathToServerBinary: string;
+};
+
+export interface StartOptions {
+  /** whether to disable outputting stderr to the parent process's stderr */
+  muteStdErr: boolean;
+}
+
+/**
+ * 
+ * @param directory directory for accounts folder
+ * @param options 
+ */
+export function startDeltaChat(directory: string, options?: Partial<SearchOptions & StartOptions> ): Promise<DeltaChatOverJsonRpcServer>
+
+
+export namespace FnTypes {
+    export type getRPCServerPath = typeof getRPCServerPath
+    export type startDeltaChat = typeof startDeltaChat
+}

deltachat-rpc-server/npm-package/index.js

@@ -0,0 +1,108 @@
+//@ts-check
+import { spawn } from "node:child_process";
+import { stat } from "node:fs/promises";
+import os from "node:os";
+import process from "node:process";
+import { ENV_VAR_NAME, PATH_EXECUTABLE_NAME } from "./src/const.js";
+import {
+  ENV_VAR_LOCATION_NOT_FOUND,
+  FAILED_TO_START_SERVER_EXECUTABLE,
+  NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR,
+  NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
+} from "./src/errors.js";
+
+import { createRequire } from "node:module";
+
+function findRPCServerInNodeModules() {
+  const arch = os.arch();
+  const operating_system = process.platform;
+  const package_name = `@deltachat/stdio-rpc-server-${operating_system}-${arch}`;
+  try {
+    const { resolve } = createRequire(import.meta.url);
+    return resolve(package_name);
+  } catch (error) {
+    console.debug("findRpcServerInNodeModules", error);
+    const require = createRequire(import.meta.url);
+    if (
+      Object.keys(require("./package.json").optionalDependencies).includes(
+        package_name
+      )
+    ) {
+      throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
+    } else {
+      throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());
+    }
+  }
+}
+
+/** @type {import("./index").FnTypes.getRPCServerPath} */
+export async function getRPCServerPath(options = {}) {
+  const { takeVersionFromPATH, disableEnvPath } = {
+    takeVersionFromPATH: false,
+    disableEnvPath: false,
+    ...options,
+  };
+  // 1. check if it is set as env var
+  if (process.env[ENV_VAR_NAME] && !disableEnvPath) {
+    try {
+      if (!(await stat(process.env[ENV_VAR_NAME])).isFile()) {
+        throw new Error(
+          `expected ${ENV_VAR_NAME} to point to the deltachat-rpc-server executable`
+        );
+      }
+    } catch (error) {
+      throw new Error(ENV_VAR_LOCATION_NOT_FOUND());
+    }
+    return process.env[ENV_VAR_NAME];
+  }
+
+  // 2. check if PATH should be used
+  if (takeVersionFromPATH) {
+    return PATH_EXECUTABLE_NAME;
+  }
+  // 3. check for prebuilds
+
+  return findRPCServerInNodeModules();
+}
+
+import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
+
+/** @type {import("./index").FnTypes.startDeltaChat} */
+export async function startDeltaChat(directory, options = {}) {
+  const pathToServerBinary = await getRPCServerPath(options);
+  const server = spawn(pathToServerBinary, {
+    env: {
+      RUST_LOG: process.env.RUST_LOG,
+      DC_ACCOUNTS_PATH: directory,
+    },
+    stdio: ["pipe", "pipe", options.muteStdErr ? "ignore" : "inherit"],
+  });
+
+  server.on("error", (err) => {
+    throw new Error(FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, err));
+  });
+  let shouldClose = false;
+
+  server.on("exit", () => {
+    if (shouldClose) {
+      return;
+    }
+    throw new Error("Server quit");
+  });
+
+  /** @type {import('./index').DeltaChatOverJsonRpcServer} */
+  //@ts-expect-error
+  const dc = new StdioDeltaChat(server.stdin, server.stdout, true);
+
+  dc.close = () => {
+    shouldClose = true;
+    if (!server.kill()) {
+      console.log("server termination failed");
+    }
+  };
+
+  //@ts-expect-error
+  dc.pathToServerBinary = pathToServerBinary;
+
+  return dc;
+}

deltachat-rpc-server/npm-package/package.json

@@ -0,0 +1,19 @@
+{
+  "license": "MPL-2.0",
+  "main": "index.js",
+  "name": "@deltachat/stdio-rpc-server",
+  "optionalDependencies": {},
+  "peerDependencies": {
+    "@deltachat/jsonrpc-client": "*"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/deltachat/deltachat-core-rust.git"
+  },
+  "scripts": {
+    "prepack": "node scripts/update_optional_dependencies_and_version.js"
+  },
+  "type": "module",
+  "types": "index.d.ts",
+  "version": "1.142.6"
+}

deltachat-rpc-server/npm-package/scripts/build_platform_package.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir
+from shutil import copy
+from src.make_package import write_package_json
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+if len(argv) < 2:
+    print("First argument should be target architecture as required by cargo")
+    exit(1)
+
+target = argv[1].strip()
+
+subprocess.run(
+    ["cargo", "build", "--release", "-p", "deltachat-rpc-server", "--target", target],
+    check=True,
+)
+
+newpath = "platform_package"
+if not path.exists(newpath):
+    makedirs(newpath)
+
+# make new folder
+
+platform_path = "platform_package/" + target
+if not path.exists(platform_path):
+    makedirs(platform_path)
+
+# copy binary it over
+
+
+def binary_path(binary_name):
+    return "../../target/" + target + "/release/" + binary_name
+
+
+my_binary_name = "deltachat-rpc-server"
+
+if not path.isfile(binary_path("deltachat-rpc-server")):
+    my_binary_name = "deltachat-rpc-server.exe"
+    if not path.isfile(binary_path("deltachat-rpc-server.exe")):
+        print("Did not find the build")
+        exit(1)
+
+my_binary_path = binary_path(my_binary_name)
+
+copy(my_binary_path, platform_path + "/" + my_binary_name)
+
+# make a package.json for it
+
+write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/make_local_dev_version.py

@@ -0,0 +1,34 @@
+# This script is for making a version of the npm packet that you can install locally
+
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir
+import re
+import json
+import tomllib
+from shutil import copy, rmtree
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+# get host target with "rustc -vV"
+output = subprocess.run(["rustc", "-vV"], capture_output=True)
+host_target = re.search('host: ([-\\w]*)', output.stdout.decode("utf-8")).group(1)
+print("host target to build for is:", host_target)
+
+# clean platform_package folder
+newpath = r'platform_package' 
+if not path.exists(newpath):
+    makedirs(newpath)
+else:
+    rmtree(path.join(path.dirname(path.abspath(__file__)), "../platform_package/"))
+    makedirs(newpath)
+
+# run build_platform_package.py with the host's target to build it
+subprocess.run(["python", "scripts/build_platform_package.py", host_target], capture_output=False, check=True)
+
+# run update_optional_dependencies_and_version.js to adjust the package / make it installable locally
+subprocess.run(["node", "scripts/update_optional_dependencies_and_version.js", "--local"], capture_output=False, check=True)
+
+# typescript / npm local package installing/linking needs that this package has it's own node_modules folder
+subprocess.run(["npm", "i"], capture_output=False, check=True)

deltachat-rpc-server/npm-package/scripts/pack_binary_for_platform.py

@@ -0,0 +1,46 @@
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir, chmod, stat
+import json
+from shutil import copy
+from src.make_package import write_package_json
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+if len(argv) < 3:
+    print("First argument should be target architecture as required by cargo")
+    print("Second argument should be the location of th built binary (binary_path)")
+    exit(1)
+
+target = argv[1].strip()
+binary_path = argv[2].strip()
+
+output = subprocess.run(["rustc","--print","target-list"], capture_output=True, check=True)
+available_targets = output.stdout.decode("utf-8")
+
+if available_targets.find(target) == -1:
+    print("target", target, "is not known / not valid")
+    exit(1)
+
+
+newpath = r'platform_package' 
+if not path.exists(newpath):
+    makedirs(newpath)
+
+# make new folder
+
+platform_path = 'platform_package/' + target
+if not path.exists(platform_path):
+    makedirs(platform_path)
+
+# copy binary it over
+
+my_binary_name = path.basename(binary_path)
+new_binary_path = platform_path + "/" + my_binary_name
+copy(binary_path, new_binary_path)
+chmod(new_binary_path, 0o555) # everyone can read & execute, nobody can write
+
+# make a package.json for it
+
+write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/src/convert_platform.py

@@ -0,0 +1,21 @@
+def convert_cpu_arch_to_npm_cpu_arch(arch):
+    if arch == "x86_64":
+        return "x64"
+    if arch == "i686":
+        return "ia32"
+    if arch == "aarch64":
+        return "arm64"
+    if arch == "armv7" or arch == "arm":
+        return "arm"
+    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.arch':", arch)
+    return arch
+
+def convert_os_to_npm_os(os):
+    if os == "windows":
+        return "win32"
+    if os == "darwin" or os == "linux":
+        return os
+    if os.startswith("android"):
+        return "android"
+    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.platform':", os)
+    return os

deltachat-rpc-server/npm-package/scripts/src/make_package.py

@@ -0,0 +1,34 @@
+import tomllib
+import json
+
+from .convert_platform import convert_cpu_arch_to_npm_cpu_arch, convert_os_to_npm_os
+
+def write_package_json(platform_path, rust_target, my_binary_name):
+    if len(rust_target.split("-")) == 3:
+        [cpu_arch, vendor, os] = rust_target.split("-")
+    else:
+        [cpu_arch, vendor, os, _env] = rust_target.split("-")
+
+    # read version
+    tomlfile = open("../../Cargo.toml", 'rb')
+    version = tomllib.load(tomlfile)['package']['version']
+
+    package_json = {
+        "name": "@deltachat/stdio-rpc-server-"
+        + convert_os_to_npm_os(os)
+        + "-"
+        + convert_cpu_arch_to_npm_cpu_arch(cpu_arch),
+        "version": version,
+        "os": [convert_os_to_npm_os(os)],
+        "cpu": [convert_cpu_arch_to_npm_cpu_arch(cpu_arch)],
+        "main": my_binary_name,
+        "license": "MPL-2.0",
+        "repository": {
+            "type": "git",
+            "url": "https://github.com/deltachat/deltachat-core-rust.git",
+        },
+    }
+
+    file = open(platform_path + "/package.json", 'w')
+    file.write(json.dumps(package_json, indent=4))
+

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -0,0 +1,64 @@
+import fs from "node:fs/promises";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const expected_cwd = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+if (process.cwd() !== expected_cwd) {
+  console.error(
+    "CWD missmatch: this script needs to be run from " + expected_cwd,
+    { actual: process.cwd(), expected: expected_cwd }
+  );
+  process.exit(1);
+}
+
+// whether to use local paths instead of npm registry version number for the prebuilds in optionalDependencies
+// useful for local development
+const is_local = process.argv.includes("--local");
+
+const package_json = JSON.parse(await fs.readFile("./package.json", "utf8"));
+
+const cargo_toml = await fs.readFile("../Cargo.toml", "utf8");
+const version = cargo_toml
+  .split("\n")
+  .find((line) => line.includes("version"))
+  .split('"')[1];
+
+const platform_packages_dir = "./platform_package";
+
+const platform_package_names = await Promise.all(
+  (await fs.readdir(platform_packages_dir)).map(async (name) => {
+    const p = JSON.parse(
+      await fs.readFile(
+        join(platform_packages_dir, name, "package.json"),
+        "utf8"
+      )
+    );
+    if (p.version !== version) {
+      console.error(
+        name,
+        "has a different version than the version of the rpc server.",
+        { rpc_server: version, platform_package: p.version }
+      );
+      throw new Error("version missmatch");
+    }
+    return { folder_name: name, package_name: p.name };
+  })
+);
+
+package_json.version = version;
+package_json.optionalDependencies = {};
+for (const { folder_name, package_name } of platform_package_names) {
+  package_json.optionalDependencies[package_name] = is_local
+    ? `file:${expected_cwd}/platform_package/${folder_name}` // npm seems to work better with an absolute path here
+    : version;
+}
+
+if (is_local) {
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] =
+    `file:${join(expected_cwd, "/../../deltachat-jsonrpc/typescript")}`;
+} else {
+  package_json.peerDependencies["@deltachat/jsonrpc-client"] = "*";
+}
+
+await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deltachat-rpc-server/npm-package/src/const.js

@@ -0,0 +1,5 @@
+//@ts-check
+
+export const PATH_EXECUTABLE_NAME = 'deltachat-rpc-server'
+
+export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"

deltachat-rpc-server/npm-package/src/errors.js

@@ -0,0 +1,41 @@
+//@ts-check
+import { ENV_VAR_NAME } from "./const.js";
+
+const cargoInstallCommand =
+  "cargo install --git https://github.com/deltachat/deltachat-core-rust deltachat-rpc-server";
+
+export function NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name) {
+  return `deltachat-rpc-server not found:
+
+- Install it with "npm i ${package_name}"
+- or download/compile deltachat-rpc-server for your platform and
+ - either put it into your PATH (for example with "${cargoInstallCommand}")
+ - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
+}
+
+export function NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR() {
+  return `deltachat-rpc-server not found:
+
+Unfortunately no prebuild is available for your system, so you need to provide deltachat-rpc-server yourself.
+
+- Download or Compile deltachat-rpc-server for your platform and
+ - either put it into your PATH (for example with "${cargoInstallCommand}")
+ - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
+}
+
+export function ENV_VAR_LOCATION_NOT_FOUND(error) {
+  return `deltachat-rpc-server not found in ${ENV_VAR_NAME}:
+
+    Error: ${error}
+
+    Content of ${ENV_VAR_NAME}: "${process.env[ENV_VAR_NAME]}"`;
+}
+
+export function FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, error) {
+  return `Failed to start server executable at '${pathToServerBinary}',
+
+  Error: ${error}
+
+Make sure the deltachat-rpc-server binary exists at this location 
+and you can start it with \`${pathToServerBinary} --version\``;
+}

deltachat-rpc-server/src/main.rs

@@ -1,3 +1,4 @@
+#![recursion_limit = "256"]
 //! Delta Chat core RPC server.
 //!
 //! It speaks JSON Lines over stdio.
@@ -10,6 +11,7 @@ use deltachat::constants::DC_VERSION_STR;
 use deltachat_jsonrpc::api::{Accounts, CommandApi};
 use futures_lite::stream::StreamExt;
 use tokio::io::{self, AsyncBufReadExt, BufReader};
+use tracing_subscriber::EnvFilter;
 use yerpc::RpcServer as _;
 
 #[cfg(target_family = "unix")]
@@ -27,6 +29,9 @@ async fn main() {
     // "For technical reasons, stdin is implemented by using an ordinary blocking read on a separate
     // thread, and it is impossible to cancel that read. This can make shutdown of the runtime hang
     // until the user presses enter."
+    if let Err(error) = &r {
+        log::error!("Fatal error: {error:#}.")
+    }
     std::process::exit(if r.is_ok() { 0 } else { 1 });
 }
 
@@ -59,7 +64,13 @@ async fn main_impl() -> Result<()> {
     #[cfg(target_family = "unix")]
     let mut sigterm = signal_unix::signal(signal_unix::SignalKind::terminate())?;
 
-    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+    // Logs from `log` crate and traces from `tracing` crate
+    // are configurable with `RUST_LOG` environment variable
+    // and go to stderr to avoid interferring with JSON-RPC using stdout.
+    tracing_subscriber::fmt()
+        .with_env_filter(EnvFilter::from_default_env())
+        .with_writer(std::io::stderr)
+        .init();
 
     let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "accounts".to_string());
     log::info!("Starting with accounts directory `{}`.", path);
@@ -68,7 +79,7 @@ async fn main_impl() -> Result<()> {
 
     log::info!("Creating JSON-RPC API.");
     let accounts = Arc::new(RwLock::new(accounts));
-    let state = CommandApi::from_arc(accounts.clone());
+    let state = CommandApi::from_arc(accounts.clone()).await;
 
     let (client, mut out_receiver) = RpcClient::new();
     let session = RpcSession::new(client.clone(), state.clone());

deltachat-time/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "deltachat-time"
+version = "1.0.0"
+description = "Time-related tools"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]

deltachat-time/src/lib.rs

@@ -0,0 +1,35 @@
+#![allow(missing_docs)]
+
+use std::sync::RwLock;
+use std::time::{Duration, SystemTime};
+
+static SYSTEM_TIME_SHIFT: RwLock<Duration> = RwLock::new(Duration::new(0, 0));
+
+/// Fake struct for mocking `SystemTime::now()` for test purposes. You still need to use
+/// `SystemTime` as a struct representing a system time.
+pub struct SystemTimeTools();
+
+impl SystemTimeTools {
+    pub const UNIX_EPOCH: SystemTime = SystemTime::UNIX_EPOCH;
+
+    pub fn now() -> SystemTime {
+        return SystemTime::now() + *SYSTEM_TIME_SHIFT.read().unwrap();
+    }
+
+    /// Simulates a system clock forward adjustment by `duration`.
+    pub fn shift(duration: Duration) {
+        *SYSTEM_TIME_SHIFT.write().unwrap() += duration;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        SystemTimeTools::shift(Duration::from_secs(60));
+        let t = SystemTimeTools::now();
+        assert!(t > SystemTime::now());
+    }
+}

deny.toml

@@ -1,5 +1,4 @@
 [advisories]
-unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
@@ -10,6 +9,16 @@ ignore = [
     # There is no fix at the time of writing this (2023-11-28).
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
+
+    # Unmaintained ansi_term
+    "RUSTSEC-2021-0139",
+
+    # Unmaintained encoding
+    "RUSTSEC-2021-0153",
+
+    # Problem in curve25519-dalek 3.2.0 used by iroh 0.4.
+    # curve25519-dalek 4.1.3 has the problem fixed.
+    "RUSTSEC-2024-0344",
 ]
 
 [bans]
@@ -18,9 +27,13 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
+     { name = "asn1-rs-derive", version = "0.4.0" },
+     { name = "asn1-rs-impl", version = "0.1.0" },
+     { name = "asn1-rs", version = "0.5.2" },
      { name = "async-channel", version = "1.9.0" },
      { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
+     { name = "base64", version = "0.21.7" },
      { name = "bitflags", version = "1.3.2" },
      { name = "block-buffer", version = "<0.10" },
      { name = "convert_case", version = "0.4.0" },
@@ -28,48 +41,77 @@ skip = [
      { name = "darling_core", version = "<0.14" },
      { name = "darling_macro", version = "<0.14" },
      { name = "darling", version = "<0.14" },
+     { name = "der_derive", version = "0.6.1" },
+     { name = "derive_more", version = "0.99.17" },
+     { name = "der-parser", version = "8.2.0" },
      { name = "der", version = "0.6.1" },
      { name = "digest", version = "<0.10" },
+     { name = "dlopen2", version = "0.4.1" },
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
      { name = "event-listener", version = "2.5.3" },
-     { name = "fd-lock", version = "3.0.13" },
+     { name = "event-listener", version = "4.0.3" },
+     { name = "fastrand", version = "1.9.0" },
+     { name = "futures-lite", version = "1.13.0" },
      { name = "getrandom", version = "<0.2" },
-     { name = "h2", version = "0.3.22" },
-     { name = "http-body", version = "0.4.5" },
-     { name = "http", version = "0.2.11" },
-     { name = "hyper", version = "0.14.27" },
+     { name = "h2", version = "0.3.26" },
+     { name = "http-body", version = "0.4.6" },
+     { name = "http", version = "0.2.12" },
+     { name = "hyper", version = "0.14.28" },
      { name = "idna", version = "0.4.0" },
+     { name = "netlink-packet-core", version = "0.5.0" },
+     { name = "netlink-packet-route", version = "0.15.0" },
+     { name = "nix", version = "0.26.4" },
+     { name = "oid-registry", version = "0.6.1" },
      { name = "pem-rfc7468", version = "0.6.0" },
+     { name = "pem", version = "1.1.1" },
      { name = "pkcs8", version = "0.9.0" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
+     { name = "rcgen", version = "<0.12.1" },
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
      { name = "ring", version = "0.16.20" },
+     { name = "rustls-pemfile", version = "1.0.4" },
+     { name = "rustls", version = "0.21.11" },
+     { name = "rustls-webpki", version = "0.101.7" },
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
-     { name = "socket2", version = "0.4.9" },
      { name = "spin", version = "<0.9.6" },
      { name = "spki", version = "0.6.0" },
+     { name = "ssh-encoding", version = "0.1.0" },
+     { name = "ssh-key", version = "0.5.1" },
+     { name = "strsim", version = "0.10.0" },
+     { name = "sync_wrapper", version = "0.1.2" },
+     { name = "synstructure", version = "0.12.6" },
      { name = "syn", version = "1.0.109" },
+     { name = "system-configuration-sys", version = "0.5.0" },
+     { name = "system-configuration", version = "0.5.1" },
      { name = "time", version = "<0.3" },
+     { name = "tokio-rustls", version = "0.24.1" },
+     { name = "toml_edit", version = "0.21.1" },
      { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
+     { name = "webpki-roots", version ="0.25.4" },
      { name = "windows_aarch64_gnullvm", version = "<0.52" },
      { name = "windows_aarch64_msvc", version = "<0.52" },
+     { name = "windows-core", version = "<0.54.0" },
      { name = "windows_i686_gnu", version = "<0.52" },
      { name = "windows_i686_msvc", version = "<0.52" },
      { name = "windows-sys", version = "<0.52" },
      { name = "windows-targets", version = "<0.52" },
      { name = "windows", version = "0.32.0" },
+     { name = "windows", version = "<0.54.0" },
      { name = "windows_x86_64_gnullvm", version = "<0.52" },
      { name = "windows_x86_64_gnu", version = "<0.52" },
      { name = "windows_x86_64_msvc", version = "<0.52" },
+     { name = "winnow", version = "0.5.40" },
+     { name = "winreg", version = "0.50.0" },
+     { name = "x509-parser", version = "<0.16.0" },
 ]
 
 
@@ -101,6 +143,4 @@ license-files = [
 github = [
        "async-email",
        "deltachat",
-       "djc",
-       "n0-computer", # iroh
 ]

flake.lock

@@ -0,0 +1,287 @@
+{
+  "nodes": {
+    "android": {
+      "inputs": {
+        "devshell": "devshell",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1712088936,
+        "narHash": "sha256-mVjeSWQiR/t4UZ9fUawY9OEPAhY1R3meYG+0oh8DUBs=",
+        "owner": "tadfisher",
+        "repo": "android-nixpkgs",
+        "rev": "2d8181caef279f19c4a33dc694723f89ffc195d4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "tadfisher",
+        "repo": "android-nixpkgs",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "android",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1711099426,
+        "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_2",
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1714112748,
+        "narHash": "sha256-jq6Cpf/pQH85p+uTwPPrGG8Ky/zUOTwMJ7mcqc5M4So=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "3ae4b908a795b6a3824d401a0702e11a7157d7e1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_3"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "naersk": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_3"
+      },
+      "locked": {
+        "lastModified": 1713520724,
+        "narHash": "sha256-CO8MmVDmqZX2FovL75pu5BvwhW+Vugc7Q6ze7Hj8heI=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "c5037590290c6c7dae2e42e7da1e247e54ed2d49",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "naersk",
+        "type": "github"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1711703276,
+        "narHash": "sha256-iMUFArF0WCatKK6RzfUJknjem0H9m4KgorO/p3Dopkk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d8fe5e6c92d0d190646fb9f1056741a229980089",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1713895582,
+        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1711668574,
+        "narHash": "sha256-u1dfs0ASQIEr1icTVrsKwg2xToIpn7ZXxW3RHfHxshg=",
+        "path": "/nix/store/9fpv0kjq9a80isa1wkkvrdqsh9dpcn05-source",
+        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_4": {
+      "locked": {
+        "lastModified": 1714076141,
+        "narHash": "sha256-Drmja/f5MRHZCskS6mvzFqxEaZMeciScCTFxWVLqWEY=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "7bb2ccd8cdc44c91edba16c48d2c8f331fb3d856",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "android": "android",
+        "fenix": "fenix",
+        "flake-utils": "flake-utils_3",
+        "naersk": "naersk",
+        "nix-filter": "nix-filter",
+        "nixpkgs": "nixpkgs_4"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1714031783,
+        "narHash": "sha256-xS/niQsq1CQPOe4M4jvVPO2cnXS/EIeRG5gIopUbk+Q=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "56bee2ddafa6177b19c631eedc88d43366553223",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,552 @@
+{
+  description = "Delta Chat core";
+  inputs = {
+    fenix.url = "github:nix-community/fenix";
+    flake-utils.url = "github:numtide/flake-utils";
+    naersk.url = "github:nix-community/naersk";
+    nix-filter.url = "github:numtide/nix-filter";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    android.url = "github:tadfisher/android-nixpkgs";
+  };
+  outputs = { self, nixpkgs, flake-utils, nix-filter, naersk, fenix, android }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        inherit (pkgs.stdenv) isDarwin;
+        fenixPkgs = fenix.packages.${system};
+        naersk' = pkgs.callPackage naersk { };
+        manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
+        androidSdk = android.sdk.${system} (sdkPkgs:
+          builtins.attrValues {
+            inherit (sdkPkgs) ndk-24-0-8215888 cmdline-tools-latest;
+          });
+        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/24.0.8215888";
+
+        rustSrc = nix-filter.lib {
+          root = ./.;
+
+          # Include only necessary files
+          # to avoid rebuilds e.g. when README.md or flake.nix changes.
+          include = [
+            ./benches
+            ./assets
+            ./Cargo.lock
+            ./Cargo.toml
+            ./CMakeLists.txt
+            ./CONTRIBUTING.md
+            ./deltachat_derive
+            ./deltachat-contact-tools
+            ./deltachat-ffi
+            ./deltachat-jsonrpc
+            ./deltachat-ratelimit
+            ./deltachat-repl
+            ./deltachat-rpc-client
+            ./deltachat-time
+            ./deltachat-rpc-server
+            ./format-flowed
+            ./release-date.in
+            ./src
+          ];
+          exclude = [
+            (nix-filter.lib.matchExt "nix")
+            "flake.lock"
+          ];
+        };
+
+        # Map from architecture name to rust targets and nixpkgs targets.
+        arch2targets = {
+          "x86_64-linux" = {
+            rustTarget = "x86_64-unknown-linux-musl";
+            crossTarget = "x86_64-unknown-linux-musl";
+          };
+          "armv7l-linux" = {
+            rustTarget = "armv7-unknown-linux-musleabihf";
+            crossTarget = "armv7l-unknown-linux-musleabihf";
+          };
+          "armv6l-linux" = {
+            rustTarget = "arm-unknown-linux-musleabihf";
+            crossTarget = "armv6l-unknown-linux-musleabihf";
+          };
+          "aarch64-linux" = {
+            rustTarget = "aarch64-unknown-linux-musl";
+            crossTarget = "aarch64-unknown-linux-musl";
+          };
+          "i686-linux" = {
+            rustTarget = "i686-unknown-linux-musl";
+            crossTarget = "i686-unknown-linux-musl";
+          };
+
+          "x86_64-darwin" = {
+            rustTarget = "x86_64-apple-darwin";
+            crossTarget = "x86_64-darwin";
+          };
+          "aarch64-darwin" = {
+            rustTarget = "aarch64-apple-darwin";
+            crossTarget = "aarch64-darwin";
+          };
+        };
+        cargoLock = {
+          lockFile = ./Cargo.lock;
+          outputHashes = {
+            "email-0.0.20" = "sha256-rV4Uzqt2Qdrfi5Ti1r+Si1c2iW1kKyWLwOgLkQ5JGGw=";
+            "encoded-words-0.2.0" = "sha256-KK9st0hLFh4dsrnLd6D8lC6pRFFs8W+WpZSGMGJcosk=";
+            "lettre-0.9.2" = "sha256-+hU1cFacyyeC9UGVBpS14BWlJjHy90i/3ynMkKAzclk=";
+          };
+        };
+        mkRustPackage = packageName:
+          naersk'.buildPackage {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            buildInputs = pkgs.lib.optionals isDarwin [
+              pkgs.darwin.apple_sdk.frameworks.SystemConfiguration
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+          };
+        pkgsWin64 = pkgs.pkgsCross.mingwW64;
+        mkWin64RustPackage = packageName:
+          let
+            rustTarget = "x86_64-pc-windows-gnu";
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              pkgsWin64.stdenv.cc
+              pkgsWin64.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+          };
+
+        pkgsWin32 = pkgs.pkgsCross.mingw32;
+        mkWin32RustPackage = packageName:
+          let
+            rustTarget = "i686-pc-windows-gnu";
+          in
+          let
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+
+            # Get rid of MCF Gthread library.
+            # See <https://github.com/NixOS/nixpkgs/issues/156343>
+            # and <https://discourse.nixos.org/t/statically-linked-mingw-binaries/38395>
+            # for details.
+            #
+            # Use DWARF-2 instead of SJLJ for exception handling.
+            winCC = pkgsWin32.buildPackages.wrapCC (
+              (pkgsWin32.buildPackages.gcc-unwrapped.override
+                ({
+                  threadsCross = {
+                    model = "win32";
+                    package = null;
+                  };
+                })).overrideAttrs (oldAttr: {
+                configureFlags = oldAttr.configureFlags ++ [
+                  "--disable-sjlj-exceptions --with-dwarf2"
+                ];
+              })
+            );
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              winCC
+              pkgsWin32.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            LD = "${winCC}/bin/${winCC.targetPrefix}cc";
+          };
+
+        mkCrossRustPackage = arch: packageName:
+          let
+            rustTarget = arch2targets."${arch}".rustTarget;
+            crossTarget = arch2targets."${arch}".crossTarget;
+            pkgsCross = import nixpkgs {
+              system = system;
+              crossSystem.config = crossTarget;
+            };
+          in
+          let
+            toolchain = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naersk-lib = pkgs.callPackage naersk {
+              cargo = toolchain;
+              rustc = toolchain;
+            };
+          in
+          naersk-lib.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = rustSrc;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+          };
+
+        androidAttrs = {
+          armeabi-v7a = {
+            cc = "armv7a-linux-androideabi19-clang";
+            rustTarget = "armv7-linux-androideabi";
+          };
+          arm64-v8a = {
+            cc = "aarch64-linux-android21-clang";
+            rustTarget = "aarch64-linux-android";
+          };
+        };
+
+        mkAndroidRustPackage = arch: packageName:
+          let
+            rustTarget = androidAttrs.${arch}.rustTarget;
+            toolchain = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naersk-lib = pkgs.callPackage naersk {
+              cargo = toolchain;
+              rustc = toolchain;
+            };
+            targetToolchain = "${androidNdkRoot}/toolchains/llvm/prebuilt/linux-x86_64";
+            targetCcName = androidAttrs.${arch}.cc;
+            targetCc = "${targetToolchain}/bin/${targetCcName}";
+          in
+          naersk-lib.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = rustSrc;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${targetCc}";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${targetCc}";
+            LD = "${targetCc}";
+          };
+
+        mkAndroidPackages = arch: {
+          "deltachat-rpc-server-${arch}-android" = mkAndroidRustPackage arch "deltachat-rpc-server";
+          "deltachat-repl-${arch}-android" = mkAndroidRustPackage arch "deltachat-repl";
+        };
+
+        mkRustPackages = arch:
+          let
+            rpc-server = mkCrossRustPackage arch "deltachat-rpc-server";
+          in
+          {
+            "deltachat-repl-${arch}" = mkCrossRustPackage arch "deltachat-repl";
+            "deltachat-rpc-server-${arch}" = rpc-server;
+            "deltachat-rpc-server-${arch}-wheel" =
+              pkgs.stdenv.mkDerivation {
+                pname = "deltachat-rpc-server-${arch}-wheel";
+                version = manifest.version;
+                src = nix-filter.lib {
+                  root = ./.;
+                  include = [
+                    "scripts/wheel-rpc-server.py"
+                    "deltachat-rpc-server/README.md"
+                    "LICENSE"
+                    "Cargo.toml"
+                  ];
+                };
+                nativeBuildInputs = [
+                  pkgs.python3
+                  pkgs.python3Packages.wheel
+                ];
+                buildInputs = [
+                  rpc-server
+                ];
+                buildPhase = ''
+                  mkdir tmp
+                  cp ${rpc-server}/bin/deltachat-rpc-server tmp/deltachat-rpc-server
+                  python3 scripts/wheel-rpc-server.py ${arch} tmp/deltachat-rpc-server
+                '';
+                installPhase = ''mkdir -p $out; cp -av deltachat_rpc_server-*.whl $out'';
+              };
+          };
+      in
+      {
+        formatter = pkgs.nixpkgs-fmt;
+
+        packages =
+          mkRustPackages "aarch64-linux" //
+          mkRustPackages "i686-linux" //
+          mkRustPackages "x86_64-linux" //
+          mkRustPackages "armv7l-linux" //
+          mkRustPackages "armv6l-linux" //
+          mkAndroidPackages "armeabi-v7a" //
+          mkAndroidPackages "arm64-v8a" //
+          mkAndroidPackages "x86" //
+          mkAndroidPackages "x86_64" // rec {
+            # Run with `nix run .#deltachat-repl foo.db`.
+            deltachat-repl = mkRustPackage "deltachat-repl";
+            deltachat-rpc-server = mkRustPackage "deltachat-rpc-server";
+
+            deltachat-repl-win64 = mkWin64RustPackage "deltachat-repl";
+            deltachat-rpc-server-win64 = mkWin64RustPackage "deltachat-rpc-server";
+            deltachat-rpc-server-win64-wheel =
+              pkgs.stdenv.mkDerivation {
+                pname = "deltachat-rpc-server-win64-wheel";
+                version = manifest.version;
+                src = nix-filter.lib {
+                  root = ./.;
+                  include = [
+                    "scripts/wheel-rpc-server.py"
+                    "deltachat-rpc-server/README.md"
+                    "LICENSE"
+                    "Cargo.toml"
+                  ];
+                };
+                nativeBuildInputs = [
+                  pkgs.python3
+                  pkgs.python3Packages.wheel
+                ];
+                buildInputs = [
+                  deltachat-rpc-server-win64
+                ];
+                buildPhase = ''
+                  mkdir tmp
+                  cp ${deltachat-rpc-server-win64}/bin/deltachat-rpc-server.exe tmp/deltachat-rpc-server.exe
+                  python3 scripts/wheel-rpc-server.py win64 tmp/deltachat-rpc-server.exe
+                '';
+                installPhase = ''mkdir -p $out; cp -av deltachat_rpc_server-*.whl $out'';
+              };
+
+            deltachat-repl-win32 = mkWin32RustPackage "deltachat-repl";
+            deltachat-rpc-server-win32 = mkWin32RustPackage "deltachat-rpc-server";
+            deltachat-rpc-server-win32-wheel =
+              pkgs.stdenv.mkDerivation {
+                pname = "deltachat-rpc-server-win32-wheel";
+                version = manifest.version;
+                src = nix-filter.lib {
+                  root = ./.;
+                  include = [
+                    "scripts/wheel-rpc-server.py"
+                    "deltachat-rpc-server/README.md"
+                    "LICENSE"
+                    "Cargo.toml"
+                  ];
+                };
+                nativeBuildInputs = [
+                  pkgs.python3
+                  pkgs.python3Packages.wheel
+                ];
+                buildInputs = [
+                  deltachat-rpc-server-win32
+                ];
+                buildPhase = ''
+                  mkdir tmp
+                  cp ${deltachat-rpc-server-win32}/bin/deltachat-rpc-server.exe tmp/deltachat-rpc-server.exe
+                  python3 scripts/wheel-rpc-server.py win32 tmp/deltachat-rpc-server.exe
+                '';
+                installPhase = ''mkdir -p $out; cp -av deltachat_rpc_server-*.whl $out'';
+              };
+            # Run `nix build .#docs` to get C docs generated in `./result/`.
+            docs =
+              pkgs.stdenv.mkDerivation {
+                pname = "docs";
+                version = manifest.version;
+                src = pkgs.lib.cleanSource ./.;
+                nativeBuildInputs = [ pkgs.doxygen ];
+                buildPhase = ''scripts/run-doxygen.sh'';
+                installPhase = ''mkdir -p $out; cp -av deltachat-ffi/html deltachat-ffi/xml $out'';
+              };
+
+            libdeltachat =
+              pkgs.stdenv.mkDerivation {
+                pname = "libdeltachat";
+                version = manifest.version;
+                src = rustSrc;
+                cargoDeps = pkgs.rustPlatform.importCargoLock cargoLock;
+
+                nativeBuildInputs = [
+                  pkgs.perl # Needed to build vendored OpenSSL.
+                  pkgs.cmake
+                  pkgs.rustPlatform.cargoSetupHook
+                  pkgs.cargo
+                ];
+                buildInputs = pkgs.lib.optionals isDarwin [
+                  pkgs.darwin.apple_sdk.frameworks.CoreFoundation
+                  pkgs.darwin.apple_sdk.frameworks.Security
+                  pkgs.darwin.apple_sdk.frameworks.SystemConfiguration
+                  pkgs.libiconv
+                ];
+
+                postInstall = ''
+                  substituteInPlace $out/include/deltachat.h \
+                    --replace __FILE__ '"${placeholder "out"}/include/deltachat.h"'
+                '';
+              };
+
+            # Source package for deltachat-rpc-server.
+            # Fake package that downloads Linux version,
+            # needed to install deltachat-rpc-server on Android with `pip`.
+            deltachat-rpc-server-source =
+              pkgs.stdenv.mkDerivation {
+                pname = "deltachat-rpc-server-source";
+                version = manifest.version;
+                src = pkgs.lib.cleanSource ./.;
+                nativeBuildInputs = [
+                  pkgs.python3
+                  pkgs.python3Packages.wheel
+                ];
+                buildPhase = ''python3 scripts/wheel-rpc-server.py source deltachat-rpc-server-${manifest.version}.tar.gz'';
+                installPhase = ''mkdir -p $out; cp -av deltachat-rpc-server-${manifest.version}.tar.gz $out'';
+              };
+
+            deltachat-rpc-client =
+              pkgs.python3Packages.buildPythonPackage {
+                pname = "deltachat-rpc-client";
+                version = manifest.version;
+                src = pkgs.lib.cleanSource ./deltachat-rpc-client;
+                format = "pyproject";
+                propagatedBuildInputs = [
+                  pkgs.python3Packages.setuptools
+                  pkgs.python3Packages.imap-tools
+                ];
+              };
+
+            deltachat-python =
+              pkgs.python3Packages.buildPythonPackage {
+                pname = "deltachat-python";
+                version = manifest.version;
+                src = pkgs.lib.cleanSource ./python;
+                format = "pyproject";
+                buildInputs = [
+                  libdeltachat
+                ];
+                nativeBuildInputs = [
+                  pkgs.pkg-config
+                ];
+                propagatedBuildInputs = [
+                  pkgs.python3Packages.setuptools
+                  pkgs.python3Packages.pkgconfig
+                  pkgs.python3Packages.cffi
+                  pkgs.python3Packages.imap-tools
+                  pkgs.python3Packages.pluggy
+                  pkgs.python3Packages.requests
+                ];
+              };
+            python-docs =
+              pkgs.stdenv.mkDerivation {
+                pname = "docs";
+                version = manifest.version;
+                src = pkgs.lib.cleanSource ./.;
+                buildInputs = [
+                  deltachat-python
+                  deltachat-rpc-client
+                  pkgs.python3Packages.breathe
+                  pkgs.python3Packages.sphinx_rtd_theme
+                ];
+                nativeBuildInputs = [ pkgs.sphinx ];
+                buildPhase = ''sphinx-build -b html -a python/doc/ dist/html'';
+                installPhase = ''mkdir -p $out; cp -av dist/html $out'';
+              };
+          };
+
+        devShells.default = let 
+          pkgs = import nixpkgs {
+            system = system;
+            overlays = [ fenix.overlays.default ];
+          };
+          in pkgs.mkShell {
+
+          buildInputs = with pkgs; [
+            (fenix.packages.${system}.complete.withComponents [
+              "cargo"
+              "clippy"
+              "rust-src"
+              "rustc"
+              "rustfmt"
+            ])
+            cargo-deny
+            rust-analyzer-nightly
+            cargo-nextest
+            perl # needed to build vendored OpenSSL
+            git-cliff
+          ];
+        };
+      }
+    );
+}

node/constants.js

@@ -29,9 +29,14 @@ module.exports = {
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
+  DC_EVENT_CHANNEL_OVERFLOW: 2400,
+  DC_EVENT_CHATLIST_CHANGED: 2300,
+  DC_EVENT_CHATLIST_ITEM_CHANGED: 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
+  DC_EVENT_CONFIG_SYNCED: 2111,
   DC_EVENT_CONNECTIVITY_CHANGED: 2100,
   DC_EVENT_CONTACTS_CHANGED: 2030,
   DC_EVENT_DELETED_BLOB_FILE: 151,
@@ -62,6 +67,7 @@ module.exports = {
   DC_EVENT_SMTP_MESSAGE_SENT: 103,
   DC_EVENT_WARNING: 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED: 2121,
+  DC_EVENT_WEBXDC_REALTIME_DATA: 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE: 2120,
   DC_GCL_ADD_ALLDONE_HINT: 4,
   DC_GCL_ADD_SELF: 2,
@@ -79,6 +85,7 @@ module.exports = {
   DC_INFO_EPHEMERAL_TIMER_CHANGED: 10,
   DC_INFO_GROUP_IMAGE_CHANGED: 3,
   DC_INFO_GROUP_NAME_CHANGED: 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL: 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED: 8,
   DC_INFO_LOCATION_ONLY: 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP: 4,
@@ -105,6 +112,7 @@ module.exports = {
   DC_MSG_IMAGE: 20,
   DC_MSG_STICKER: 23,
   DC_MSG_TEXT: 10,
+  DC_MSG_VCARD: 90,
   DC_MSG_VIDEO: 50,
   DC_MSG_VIDEOCHAT_INVITATION: 70,
   DC_MSG_VOICE: 41,
@@ -112,11 +120,15 @@ module.exports = {
   DC_PROVIDER_STATUS_BROKEN: 3,
   DC_PROVIDER_STATUS_OK: 1,
   DC_PROVIDER_STATUS_PREPARATION: 2,
+  DC_PUSH_CONNECTED: 2,
+  DC_PUSH_HEARTBEAT: 1,
+  DC_PUSH_NOT_CONNECTED: 0,
   DC_QR_ACCOUNT: 250,
   DC_QR_ADDR: 320,
   DC_QR_ASK_VERIFYCONTACT: 200,
   DC_QR_ASK_VERIFYGROUP: 202,
   DC_QR_BACKUP: 251,
+  DC_QR_BACKUP2: 252,
   DC_QR_ERROR: 400,
   DC_QR_FPR_MISMATCH: 220,
   DC_QR_FPR_OK: 210,
@@ -165,6 +177,7 @@ module.exports = {
   DC_STR_CONFIGURATION_FAILED: 84,
   DC_STR_CONNECTED: 107,
   DC_STR_CONNTECTING: 108,
+  DC_STR_CONTACT: 200,
   DC_STR_CONTACT_NOT_VERIFIED: 36,
   DC_STR_CONTACT_SETUP_CHANGED: 37,
   DC_STR_CONTACT_VERIFIED: 35,
@@ -225,11 +238,13 @@ module.exports = {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU: 124,
   DC_STR_IMAGE: 9,
   DC_STR_INCOMING_MESSAGES: 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL: 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY: 111,
   DC_STR_LOCATION: 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER: 137,
   DC_STR_LOCATION_ENABLED_BY_YOU: 136,
   DC_STR_MESSAGES: 114,
+  DC_STR_MESSAGE_ADD_MEMBER: 173,
   DC_STR_MSGACTIONBYME: 63,
   DC_STR_MSGACTIONBYUSER: 62,
   DC_STR_MSGADDMEMBER: 17,
@@ -249,12 +264,15 @@ module.exports = {
   DC_STR_PARTIAL_DOWNLOAD_MSG_BODY: 99,
   DC_STR_PART_OF_TOTAL_USED: 116,
   DC_STR_QUOTA_EXCEEDING_MSG_BODY: 98,
+  DC_STR_REACTED_BY: 177,
   DC_STR_READRCPT: 31,
   DC_STR_READRCPT_MAILBODY: 32,
   DC_STR_REMOVE_MEMBER_BY_OTHER: 131,
   DC_STR_REMOVE_MEMBER_BY_YOU: 130,
   DC_STR_REPLY_NOUN: 90,
   DC_STR_SAVED_MESSAGES: 69,
+  DC_STR_SECUREJOIN_WAIT: 190,
+  DC_STR_SECUREJOIN_WAIT_TIMEOUT: 191,
   DC_STR_SECURE_JOIN_GROUP_QR_DESC: 120,
   DC_STR_SECURE_JOIN_REPLIES: 118,
   DC_STR_SECURE_JOIN_STARTED: 117,
@@ -276,6 +294,7 @@ module.exports = {
   DC_STR_VIDEOCHAT_INVITE_MSG_BODY: 83,
   DC_STR_VOICEMESSAGE: 7,
   DC_STR_WELCOME_MESSAGE: 71,
+  DC_STR_YOU_REACTED: 176,
   DC_TEXT1_DRAFT: 1,
   DC_TEXT1_SELF: 3,
   DC_TEXT1_USERNAME: 2,

node/events.js

@@ -34,6 +34,12 @@ module.exports = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
-  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
+  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
+  2300: 'DC_EVENT_CHATLIST_CHANGED',
+  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2400: 'DC_EVENT_CHANNEL_OVERFLOW'
 }

node/lib/constants.ts

@@ -29,9 +29,14 @@ export enum C {
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
+  DC_EVENT_CHANNEL_OVERFLOW = 2400,
+  DC_EVENT_CHATLIST_CHANGED = 2300,
+  DC_EVENT_CHATLIST_ITEM_CHANGED = 2301,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
+  DC_EVENT_CONFIG_SYNCED = 2111,
   DC_EVENT_CONNECTIVITY_CHANGED = 2100,
   DC_EVENT_CONTACTS_CHANGED = 2030,
   DC_EVENT_DELETED_BLOB_FILE = 151,
@@ -62,6 +67,7 @@ export enum C {
   DC_EVENT_SMTP_MESSAGE_SENT = 103,
   DC_EVENT_WARNING = 300,
   DC_EVENT_WEBXDC_INSTANCE_DELETED = 2121,
+  DC_EVENT_WEBXDC_REALTIME_DATA = 2150,
   DC_EVENT_WEBXDC_STATUS_UPDATE = 2120,
   DC_GCL_ADD_ALLDONE_HINT = 4,
   DC_GCL_ADD_SELF = 2,
@@ -79,6 +85,7 @@ export enum C {
   DC_INFO_EPHEMERAL_TIMER_CHANGED = 10,
   DC_INFO_GROUP_IMAGE_CHANGED = 3,
   DC_INFO_GROUP_NAME_CHANGED = 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL = 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED = 8,
   DC_INFO_LOCATION_ONLY = 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP = 4,
@@ -105,6 +112,7 @@ export enum C {
   DC_MSG_IMAGE = 20,
   DC_MSG_STICKER = 23,
   DC_MSG_TEXT = 10,
+  DC_MSG_VCARD = 90,
   DC_MSG_VIDEO = 50,
   DC_MSG_VIDEOCHAT_INVITATION = 70,
   DC_MSG_VOICE = 41,
@@ -112,11 +120,15 @@ export enum C {
   DC_PROVIDER_STATUS_BROKEN = 3,
   DC_PROVIDER_STATUS_OK = 1,
   DC_PROVIDER_STATUS_PREPARATION = 2,
+  DC_PUSH_CONNECTED = 2,
+  DC_PUSH_HEARTBEAT = 1,
+  DC_PUSH_NOT_CONNECTED = 0,
   DC_QR_ACCOUNT = 250,
   DC_QR_ADDR = 320,
   DC_QR_ASK_VERIFYCONTACT = 200,
   DC_QR_ASK_VERIFYGROUP = 202,
   DC_QR_BACKUP = 251,
+  DC_QR_BACKUP2 = 252,
   DC_QR_ERROR = 400,
   DC_QR_FPR_MISMATCH = 220,
   DC_QR_FPR_OK = 210,
@@ -165,6 +177,7 @@ export enum C {
   DC_STR_CONFIGURATION_FAILED = 84,
   DC_STR_CONNECTED = 107,
   DC_STR_CONNTECTING = 108,
+  DC_STR_CONTACT = 200,
   DC_STR_CONTACT_NOT_VERIFIED = 36,
   DC_STR_CONTACT_SETUP_CHANGED = 37,
   DC_STR_CONTACT_VERIFIED = 35,
@@ -225,11 +238,13 @@ export enum C {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU = 124,
   DC_STR_IMAGE = 9,
   DC_STR_INCOMING_MESSAGES = 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL = 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY = 111,
   DC_STR_LOCATION = 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER = 137,
   DC_STR_LOCATION_ENABLED_BY_YOU = 136,
   DC_STR_MESSAGES = 114,
+  DC_STR_MESSAGE_ADD_MEMBER = 173,
   DC_STR_MSGACTIONBYME = 63,
   DC_STR_MSGACTIONBYUSER = 62,
   DC_STR_MSGADDMEMBER = 17,
@@ -249,12 +264,15 @@ export enum C {
   DC_STR_PARTIAL_DOWNLOAD_MSG_BODY = 99,
   DC_STR_PART_OF_TOTAL_USED = 116,
   DC_STR_QUOTA_EXCEEDING_MSG_BODY = 98,
+  DC_STR_REACTED_BY = 177,
   DC_STR_READRCPT = 31,
   DC_STR_READRCPT_MAILBODY = 32,
   DC_STR_REMOVE_MEMBER_BY_OTHER = 131,
   DC_STR_REMOVE_MEMBER_BY_YOU = 130,
   DC_STR_REPLY_NOUN = 90,
   DC_STR_SAVED_MESSAGES = 69,
+  DC_STR_SECUREJOIN_WAIT = 190,
+  DC_STR_SECUREJOIN_WAIT_TIMEOUT = 191,
   DC_STR_SECURE_JOIN_GROUP_QR_DESC = 120,
   DC_STR_SECURE_JOIN_REPLIES = 118,
   DC_STR_SECURE_JOIN_STARTED = 117,
@@ -276,6 +294,7 @@ export enum C {
   DC_STR_VIDEOCHAT_INVITE_MSG_BODY = 83,
   DC_STR_VOICEMESSAGE = 7,
   DC_STR_WELCOME_MESSAGE = 71,
+  DC_STR_YOU_REACTED = 176,
   DC_TEXT1_DRAFT = 1,
   DC_TEXT1_SELF = 3,
   DC_TEXT1_USERNAME = 2,
@@ -319,6 +338,12 @@ export const EventId2EventName: { [key: number]: string } = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2150: 'DC_EVENT_WEBXDC_REALTIME_DATA',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
+  2300: 'DC_EVENT_CHATLIST_CHANGED',
+  2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2400: 'DC_EVENT_CHANNEL_OVERFLOW',
 }