chore(release): prepare for 1.133.2

close #5196

.github/workflows/ci.yml

@@ -14,8 +14,7 @@ on:
   pull_request:
   push:
     branches:
-      - master
-      - stable
+      - main
 
 env:
   RUSTFLAGS: -Dwarnings
@@ -25,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.73.0
+      RUSTUP_TOOLCHAIN: 1.75.0
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -39,10 +38,6 @@ jobs:
       - name: Check
         run: cargo check --workspace --all-targets --all-features
 
-      # Check with musl libc target which is used for `deltachat-rpc-server` releases.
-      - name: Check musl
-        run: scripts/zig-musl-check.sh
-
   cargo_deny:
     name: cargo deny
     runs-on: ubuntu-latest
@@ -81,19 +76,15 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.73.0
+            rust: 1.75.0
           - os: windows-latest
-            rust: 1.73.0
+            rust: 1.75.0
           - os: macos-latest
-            rust: 1.73.0
+            rust: 1.75.0
 
-          # Minimum Supported Rust Version = 1.65.0
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
+          # Minimum Supported Rust Version = 1.70.0
           - os: ubuntu-latest
-            rust: 1.65.0
+            rust: 1.70.0
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -106,6 +97,8 @@ jobs:
         uses: swatinem/rust-cache@v2
 
       - name: Tests
+        env:
+          RUST_BACKTRACE: 1
         run: cargo test --workspace
 
       - name: Test cargo vendor
@@ -137,7 +130,7 @@ jobs:
     name: Build deltachat-rpc-server
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
@@ -152,7 +145,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
-          path: target/debug/deltachat-rpc-server
+          path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
           retention-days: 1
 
   python_lint:
@@ -173,8 +166,8 @@ jobs:
         working-directory: deltachat-rpc-client
         run: tox -e lint
 
-  python_tests:
-    name: Python tests
+  cffi_python_tests:
+    name: CFFI Python tests
     needs: ["c_library", "python_lint"]
     strategy:
       fail-fast: false
@@ -218,14 +211,14 @@ jobs:
 
       - name: Run python tests
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
         run: tox -e mypy,doc,py
 
-  aysnc_python_tests:
-    name: Async Python tests
+  rpc_python_tests:
+    name: JSON-RPC Python tests
     needs: ["python_lint", "rpc_server"]
     strategy:
       fail-fast: false
@@ -235,6 +228,8 @@ jobs:
             python: 3.12
           - os: macos-latest
             python: 3.12
+          - os: windows-latest
+            python: 3.12
 
           # PyPy tests
           - os: ubuntu-latest
@@ -265,13 +260,20 @@ jobs:
           path: target/debug
 
       - name: Make deltachat-rpc-server executable
+        if: ${{ matrix.os != 'windows-latest' }}
         run: chmod +x target/debug/deltachat-rpc-server
 
       - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.os != 'windows-latest' }}
         run: echo ${{ github.workspace }}/target/debug >> $GITHUB_PATH
 
+      - name: Add deltachat-rpc-server to path
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: |
+          "${{ github.workspace }}/target/debug" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
       - name: Run deltachat-rpc-client tests
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/deltachat-rpc-server.yml

@@ -26,35 +26,17 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Build
+      - name: Install ziglang
+        run: pip install wheel ziglang==0.11.0
+
+      - name: Build deltachat-rpc-server binaries
         run: sh scripts/zig-rpc-server.sh
 
-      - name: Upload x86_64 binary
+      - name: Upload dist directory with Linux binaries
         uses: actions/upload-artifact@v3
         with:
-          name: deltachat-rpc-server-x86_64
-          path: target/x86_64-unknown-linux-musl/release/deltachat-rpc-server
-          if-no-files-found: error
-
-      - name: Upload i686 binary
-        uses: actions/upload-artifact@v3
-        with:
-          name: deltachat-rpc-server-i686
-          path: target/i686-unknown-linux-musl/release/deltachat-rpc-server
-          if-no-files-found: error
-
-      - name: Upload aarch64 binary
-        uses: actions/upload-artifact@v3
-        with:
-          name: deltachat-rpc-server-aarch64
-          path: target/aarch64-unknown-linux-musl/release/deltachat-rpc-server
-          if-no-files-found: error
-
-      - name: Upload armv7 binary
-        uses: actions/upload-artifact@v3
-        with:
-          name: deltachat-rpc-server-armv7
-          path: target/armv7-unknown-linux-musleabihf/release/deltachat-rpc-server
+          name: linux
+          path: dist/
           if-no-files-found: error
 
   build_windows:
@@ -92,39 +74,87 @@ jobs:
 
   build_macos:
     name: Build deltachat-rpc-server for macOS
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: x86_64
+          - arch: aarch64
+
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v3
 
       - name: Setup rust target
-        run: rustup target add x86_64-apple-darwin
+        run: rustup target add ${{ matrix.arch }}-apple-darwin
 
       - name: Build
-        run: cargo build --release --package deltachat-rpc-server --target x86_64-apple-darwin --features vendored
+        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.arch }}-apple-darwin --features vendored
 
       - name: Upload binary
         uses: actions/upload-artifact@v3
         with:
-          name: deltachat-rpc-server-x86_64-macos
-          path: target/x86_64-apple-darwin/release/deltachat-rpc-server
+          name: deltachat-rpc-server-${{ matrix.arch }}-macos
+          path: target/${{ matrix.arch }}-apple-darwin/release/deltachat-rpc-server
           if-no-files-found: error
 
   publish:
-    name: Upload binaries to the release
+    name: Build wheels and upload binaries to the release
     needs: ["build_linux", "build_windows", "build_macos"]
     permissions:
       contents: write
     runs-on: "ubuntu-latest"
     steps:
-      - name: Download built binaries
-        uses: "actions/download-artifact@v3"
+      - uses: actions/checkout@v3
 
-      - name: Compose dist/ directory
+      - name: Download Linux binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: linux
+          path: dist/
+
+      - name: Download win32 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: deltachat-rpc-server-win32.exe
+          path: deltachat-rpc-server-win32.exe.d
+
+      - name: Download win64 binary
+        uses: actions/download-artifact@v3
+        with:
+          name: deltachat-rpc-server-win64.exe
+          path: deltachat-rpc-server-win64.exe.d
+
+      - name: Download macOS binary for x86_64
+        uses: actions/download-artifact@v3
+        with:
+          name: deltachat-rpc-server-x86_64-macos
+          path: deltachat-rpc-server-x86_64-macos.d
+
+      - name: Download macOS binary for aarch64
+        uses: actions/download-artifact@v3
+        with:
+          name: deltachat-rpc-server-aarch64-macos
+          path: deltachat-rpc-server-aarch64-macos.d
+
+      - name: Flatten dist/ directory
         run: |
-          mkdir dist
-          for x in x86_64 i686 aarch64 armv7 win32.exe win64.exe x86_64-macos; do
-            mv "deltachat-rpc-server-$x"/* "dist/deltachat-rpc-server-$x"
-          done
+          mv deltachat-rpc-server-win32.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
+          mv deltachat-rpc-server-win64.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
+          mv deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-macos
+          mv deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-macos
+
+      # Python 3.11 is needed for tomllib used in scripts/wheel-rpc-server.py
+      - name: Install python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - name: Install wheel
+        run: pip install wheel
+
+      - name: Build deltachat-rpc-server Python wheels and source package
+        run: scripts/wheel-rpc-server.py
 
       - name: List downloaded artifacts
         run: ls -l dist/

.github/workflows/jsonrpc-client-npm-package.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/jsonrpc.yml

@@ -2,9 +2,9 @@ name: JSON-RPC API Test
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 env:
   CARGO_TERM_COLOR: always
@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
       - name: Add Rust cache
         uses: Swatinem/rust-cache@v2
       - name: npm install
@@ -31,7 +31,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/node-docs.yml

@@ -8,7 +8,7 @@ name: Generate & upload node.js documentation
 on:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   generate:
@@ -16,10 +16,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use Node.js 16.x
+      - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:
-          node-version: 16.x
+          node-version: 18.x
 
       - name: npm install and generate documentation
         working-directory: node

.github/workflows/node-package.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV
@@ -66,10 +66,8 @@ jobs:
     runs-on: ubuntu-latest
 
     # Build Linux prebuilds inside a container with old glibc for backwards compatibility.
-    # Debian 10 contained glibc 2.28 at the time of the writing (2023-06-04): https://packages.debian.org/buster/libc6
-    # Ubuntu 18.04 is at the End of Standard Support since June 2023, but it contains glibc 2.27,
-    # so we are using it to support Ubuntu 18.04 setups that are still not upgraded.
-    container: ubuntu:18.04
+    # Debian 10 contained glibc 2.28: https://packages.debian.org/buster/libc6
+    container: debian:10
     steps:
       # Working directory is owned by 1001:1001 by default.
       # Change it to our user.
@@ -80,7 +78,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - run: apt-get update
 
       # Python is needed for node-gyp
@@ -143,7 +141,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v2
         with:
-          node-version: "16"
+          node-version: "18"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/node-tests.yml

@@ -13,7 +13,7 @@ on:
   pull_request:
   push:
     branches:
-      - master
+      - main
 
 jobs:
   tests:
@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "16"
+          node-version: "18"
       - name: System info
         run: |
           rustc -vV
@@ -63,5 +63,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/upload-docs.yml

@@ -3,8 +3,7 @@ name: Build & Deploy Documentation on rs.delta.chat
 on:
   push:
     branches:
-      - master
-      - docs-gh-action
+      - main
 
 jobs:
   build:

.github/workflows/upload-ffi-docs.yml

@@ -7,8 +7,7 @@ name: Build & Deploy Documentation on cffi.delta.chat
 on:
   push:
     branches:
-      - master
-      - docs-gh-action
+      - main
 
 jobs:
   build:

.github/workflows/upload-python-docs.yml

@@ -0,0 +1,25 @@
+name: Build & Deploy Documentation on py.delta.chat
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - name: Build Python documentation
+        run: scripts/build-python-docs.sh
+      - name: Upload to py.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: delta
+          KEY: ${{ secrets.CODESPEAK_KEY }}
+          HOST: "lists.codespeak.net"
+          SOURCE: "dist/html/"
+          TARGET: "/home/delta/build/master"

.gitignore

@@ -1,6 +1,7 @@
 /target
 **/*.rs.bk
 /build
+/dist
 
 # ignore vi temporaries
 *~

CHANGELOG.md

@@ -1,5 +1,546 @@
 # Changelog
 
+## [1.133.2] - 2024-01-24
+
+### Fixes
+
+- Downgrade OpenSSL from 3.2.0 to 3.1.4 ([#5206](https://github.com/deltachat/deltachat-core-rust/issues/5206))
+- No new chats for MDNs with alias ([#5196](https://github.com/deltachat/deltachat-core-rust/issues/5196)) ([#5199](https://github.com/deltachat/deltachat-core-rust/pull/5199)).
+
+## [1.133.1] - 2024-01-21
+
+### API-Changes
+
+- Add `is_bot` to cffi and jsonrpc ([#5197](https://github.com/deltachat/deltachat-core-rust/pull/5197)).
+
+### Features / Changes
+
+- Add system message when provider does not allow unencrypted messages ([#5195](https://github.com/deltachat/deltachat-core-rust/pull/5195)).
+
+### Fixes
+
+- `Chat::send_msg`: Remove encryption-related params from already sent message. This allows to send received encrypted `dc_msg_t` object to unencrypted chat, e.g. in a Python bot.
+- Set message download state to Failure on IMAP errors. This avoids partially downloaded messages getting stuck in "Downloading..." state without actually being in a download queue.
+- BCC-to-self even if server deletion is set to "at once". This is a workaround for SMTP servers which do not return response in time, BCC-self works as a confirmation that message was sent out successfully and does not need more retries.
+- node: Run tests with native ESM modules instead of `esm` ([#5194](https://github.com/deltachat/deltachat-core-rust/pull/5194)).
+- Use Quoted-Printable MIME encoding for the text part ([#3986](https://github.com/deltachat/deltachat-core-rust/pull/3986)).
+
+### Tests
+
+- python: Add `get_protected_chat` to testplugin.py.
+
+## [1.133.0] - 2024-01-14
+
+### Features / Changes
+
+- Securejoin protocol implementation refinements
+  - Track forward and backward verification separately ([#5089](https://github.com/deltachat/deltachat-core-rust/pull/5089)) to avoid inconsistent states.
+  - Mark 1:1 chat as verified for Bob early. 1:1 chat with Alice is verified as soon as Alice's key is verified rather than at the end of the protocol.
+- Put Message-ID into hidden headers and take it from there on receiver ([#4798](https://github.com/deltachat/deltachat-core-rust/pull/4798)). This works around servers which generate their own Message-ID and overwrite the one generated by Delta Chat.
+- deltachat-repl: Enable INFO logging by default and add timestamps.
+- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elments based on the configuration key which is a part of the event.
+- Sync contact creation/rename across devices ([#5163](https://github.com/deltachat/deltachat-core-rust/pull/5163)).
+- Encrypt MDNs ([#5175](https://github.com/deltachat/deltachat-core-rust/pull/5175)).
+- Only try to configure non-strict TLS checks if explicitly set ([#5181](https://github.com/deltachat/deltachat-core-rust/pull/5181)).
+
+### Build system
+
+- Use released version of iroh 0.4.2 for "setup second device" feature.
+
+### CI
+
+- Update to Rust 1.75.0.
+- Downgrade `chai` from 4.4.0 to 4.3.10.
+
+### Documentation
+
+- Add a link <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html> to autoconfig RFC draft.
+- Update securejoin link in `standards.md` from <https://countermitm.readthedocs.io/> to <https://securejoin.readthedocs.io>.
+- Restore "Constants" page in Doxygen >=1.9.8
+
+### Fixes
+
+- imap: Limit the rate of LOGIN attempts rather than connection attempts. This is to avoid having to wait for rate limiter right after switching from a bad or offline network to a working network while still guarding against reconnection loop.
+- Do not ignore `peerstate.save_to_db()` errors.
+- securejoin: Mark 1:1s as protected regardless of the Config::VerifiedOneOnOneChats.
+- Delete received outgoing messages from SMTP queue ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- imap: Fail fast on `LIST` errors to avoid busy loop when connection is lost.
+- Split SMTP jobs already in `chat::create_send_msg_jobs()` ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- Do not remove contents from unencrypted [Schleuder](https://schleuder.org/) mailing lists messages.
+- Reset message error when scheduling resending ([#5119](https://github.com/deltachat/deltachat-core-rust/pull/5119)).
+- Emit events more reliably when starting and stopping I/O ([#5101](https://github.com/deltachat/deltachat-core-rust/pull/5101)).
+- Fix timestamp of chat protection info message for correct message ordering after restoring a backup ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+### Refactor
+
+- sql: Recreate `config` table with UNIQUE constraint.
+- sql: Recreate `keypairs` table to remove unused `addr` and `created` fields and move `is_default` flag to `config` table.
+- Send `Secure-Join-Fingerprint` only in `*-request-with-auth`.
+
+### Tests
+
+- Test joining non-protected group.
+- Test that read receipts don't degrade encryption.
+- Test that changing default private key breaks backward verification.
+- Test recovery from lost vc-contact-confirm.
+- Use `wait_for_incoming_msg_event()` more.
+
+## [1.132.1] - 2023-12-12
+
+### Features / Changes
+
+- Add "From:" to protected headers for signed-only messages.
+- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
+
+### Fixes
+
+- Add padlock to empty part if the whole message is empty.
+- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
+- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
+
+### API-Changes
+
+- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
+
+### CI
+
+- Update to Rust 1.74.1.
+
+## [1.132.0] - 2023-12-06
+
+### Features / Changes
+
+- Increase TCP timeouts from 30 to 60 seconds.
+
+### Fixes
+
+- Don't sort message creating a protected group over a protection message ([#4963](https://github.com/deltachat/deltachat-core-rust/pull/4963)).
+- Do not lock accounts.toml on iOS.
+- Protect groups even if some members are not verified and add `test_securejoin_after_contact_resetup` regression test.
+
+## [1.131.9] - 2023-12-02
+
+### API-Changes
+
+- Remove `dc_get_http_response()`, `dc_http_response_get_mimetype()`, `dc_http_response_get_encoding()`, `dc_http_response_get_blob()`, `dc_http_response_get_size()`, `dc_http_response_unref()` and `dc_http_response_t` from cffi.
+- Deprecate CFFI APIs `dc_send_reaction()`, `dc_get_msg_reactions()`, `dc_reactions_get_contacts()`, `dc_reactions_get_by_contact_id()`, `dc_reactions_unref` and `dc_reactions_t`.
+- Make `Contact.is_verified()` return bool.
+
+### Build system
+
+- Switch from fork of iroh to iroh 0.4.2 pre-release.
+
+### Features / Changes
+
+- Send `Chat-Verified` headers in 1:1 chats.
+- Ratelimit IMAP connections ([#4940](https://github.com/deltachat/deltachat-core-rust/pull/4940)).
+- Remove receiver limit on `.xdc` size.
+- Don't affect MimeMessage with "From" and secured headers from encrypted unsigned messages.
+- Sync `Config::{MdnsEnabled,ShowEmails}` across devices ([#4954](https://github.com/deltachat/deltachat-core-rust/pull/4954)).
+- Sync `Config::Displayname` across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- `Chat::rename_ex`: Don't send sync message if usual message is sent.
+
+### Fixes
+
+- Lock the database when INSERTing a webxdc update, avoid "Database is locked" errors.
+- Use keyring with all private keys when decrypting a message ([#5046](https://github.com/deltachat/deltachat-core-rust/pull/5046)).
+
+### Tests
+
+- Make Result-returning tests produce a line number.
+- Add `test_utils::sync()`.
+- Test inserting lots of webxdc updates.
+- Split `test_sync_alter_chat()` into smaller tests.
+
+## [1.131.8] - 2023-11-27
+
+### Features / Changes
+
+- webxdc: Add unique IDs to status updates sent outside and deduplicate based on IDs.
+
+### Fixes
+
+- Allow IMAP servers not returning UIDNEXT on SELECT and STATUS such as mail.163.com.
+- Use the correct securejoin strings used in the UI, remove old TODO ([#5047](https://github.com/deltachat/deltachat-core-rust/pull/5047)).
+- Do not emit events about webxdc update events logged into debug log webxdc.
+
+### Tests
+
+- Check that `receive_status_update` has forward compatibility and unique webxdc IDs will be ignored by previous Delta Chat versions.
+
+## [1.131.7] - 2023-11-24
+
+### Fixes
+
+- Revert "fix: check UIDNEXT with a STATUS command before going IDLE". This attempts to fix mail.163.com which has broken STATUS command.
+
+## [1.131.6] - 2023-11-21
+
+### Fixes
+
+- Fail fast if IMAP FETCH cannot be parsed instead of getting stuck in infinite loop.
+
+### Documentation
+
+- Generate deltachat-rpc-client documentation and publish it to <https://py.delta.chat>.
+
+## [1.131.5] - 2023-11-20
+
+### API-Changes
+
+- deltachat-rpc-client: Add `Message.get_sender_contact()`.
+- Turn `ContactAddress` into an owned type.
+
+### Features / Changes
+
+- Lowercase addresses in Autocrypt and Autocrypt-Gossip headers.
+- Lowercase the address in member added/removed messages.
+- Lowercase `addr` when it is set.
+- Do not replace the message with an error in square brackets when the sender is not a member of the protected group.
+
+### Fixes
+
+- `Chat::sync_contacts()`: Fetch contact addresses in a single query.
+- `Chat::rename_ex()`: Sync improved chat name to other devices.
+- Recognize `Chat-Group-Member-Added` of self case-insensitively.
+- Compare verifier addr to peerstate addr case-insensitively.
+
+### Tests
+
+- Port [Secure-Join](https://securejoin.readthedocs.io/) tests to JSON-RPC.
+
+### CI
+
+- Test with Rust 1.74.
+
+
+## [1.131.4] - 2023-11-16
+
+### Documentation
+
+- Document DC_DOWNLOAD_UNDECIPHERABLE.
+
+### Fixes
+
+- Always add "Member added" as system message.
+
+## [1.131.3] - 2023-11-15
+
+### Fixes
+
+- Update async-imap to 0.9.4 which does not ignore EOF on FETCH.
+- Reset gossiped timestamp on securejoin.
+- sync: Ignore unknown sync items to provide forward compatibility and avoid creating empty message bubbles.
+- sync: Skip sync when chat name is set to the current one.
+- Return connectivity HTML with an error when IO is stopped.
+
+## [1.131.2] - 2023-11-14
+
+### API-Changes
+
+- deltachat-rpc-client: add `Account.get_chat_by_contact()`.
+
+### Features / Changes
+
+- Do not post "... verified" messages on QR scan success.
+- Never drop better message from `apply_group_changes()`.
+
+### Fixes
+
+- Assign MDNs to the trash chat early to prevent received MDNs from creating or unblocking 1:1 chats.
+- Allow to securejoin groups when 1:1 chat with the inviter is a contact request.
+- Add "setup changed" message for verified key before the message.
+- Ignore special chats when calculating similar chats.
+
+## [1.131.1] - 2023-11-13
+
+### Fixes
+
+- Do not skip actual message parts when group change messages are inserted.
+
+## [1.131.0] - 2023-11-13
+
+### Features / Changes
+
+- Sync chat contacts across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync creating broadcast lists across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Sync Chat::name across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+- Multi-device broadcast lists ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
+
+### Fixes
+
+- Encode chat name in the `List-ID` header to avoid SMTPUTF8 errors.
+- Ignore errors from generating sync messages.
+- `Context::execute_sync_items`: Ignore all errors ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Allow to send unverified securejoin messages to protected chats ([#4982](https://github.com/deltachat/deltachat-core-rust/pull/4982)).
+
+## [1.130.0] - 2023-11-10
+
+### API-Changes
+
+- Emit JoinerProgress(1000) event when Bob verifies Alice.
+- JSON-RPC: add `ContactObject.is_profile_verified` property.
+- Hide `ChatId::get_for_contact()` from public API.
+
+### Features / Changes
+
+- Add secondary verified key.
+- Add info messages about implicitly added members.
+- Treat reset state as encryption not preferred.
+- Grow sleep durations on errors in Imap::fake_idle() ([#4424](https://github.com/deltachat/deltachat-core-rust/pull/4424)).
+
+### Fixes
+
+- Mark 1:1 chat as protected when joining a group.
+- Raise lower auto-download limit to 160k.
+- Remove `Reporting-UA` from read receipts.
+- Do not apply group changes to special chats. Avoid adding members to the trash chat.
+- imap: make `UidGrouper` robust against duplicate UIDs.
+- Do not return hidden chat from `dc_get_chat_id_by_contact_id`.
+- Smtp_loop(): Don't grow timeout if interrupted early ([#4833](https://github.com/deltachat/deltachat-core-rust/pull/4833)).
+
+### Refactor
+
+- imap: Do not FETCH right after `scan_folders()`.
+- deltachat-rpc-client: Use `itertools` instead of `Lock` for thread-safe request ID generation.
+
+### Tests
+
+- Remove unused `--liveconfig` option.
+- Test chatlist can load for corrupted chats ([#4979](https://github.com/deltachat/deltachat-core-rust/pull/4979)).
+
+### Miscellaneous Tasks
+
+- Update provider-db ([#4949](https://github.com/deltachat/deltachat-core-rust/pull/4949)).
+
+## [1.129.1] - 2023-11-06
+
+### Fixes
+
+- Update tokio-imap to fix Outlook STATUS parsing bug.
+- deltachat-rpc-client: Add the Lock around request ID.
+- `apply_group_changes`: Don't implicitly delete members locally, add absent ones instead ([#4934](https://github.com/deltachat/deltachat-core-rust/pull/4934)).
+- Partial messages do not change group state ([#4900](https://github.com/deltachat/deltachat-core-rust/pull/4900)).
+
+### Tests
+
+- Group chats device synchronisation.
+
+## [1.129.0] - 2023-11-06
+
+### API-Changes
+
+- Add JSON-RPC `get_chat_id_by_contact_id` API ([#4918](https://github.com/deltachat/deltachat-core-rust/pull/4918)).
+- [**breaking**] Remove deprecated `get_verifier_addr`.
+
+### Features / Changes
+
+- Sync chat `Blocked` state, chat visibility, chat mute duration and contact blocked status across devices ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+- Add 'group created instructions' as info message ([#4916](https://github.com/deltachat/deltachat-core-rust/pull/4916)).
+- Add hardcoded fallback DNS cache.
+
+### Fixes
+
+- Switch to `EncryptionPreference::Mutual` on a receipt of encrypted+signed message ([#4707](https://github.com/deltachat/deltachat-core-rust/pull/4707)).
+- imap: Check UIDNEXT with a STATUS command before going IDLE.
+- Allow to change verified key via "member added" message.
+- json-rpc: Return verifier even if the contact is not "verified" (Autocrypt key does not equal Secure-Join key).
+
+### Documentation
+
+- Refine `Contact::get_verifier_id` and `Contact::is_verified` documentation ([#4922](https://github.com/deltachat/deltachat-core-rust/pull/4922)).
+- Contact profile view should not use `dc_contact_is_verified()`.
+- Remove documentation for non-existing `dc_accounts_new` `os_name` param.
+
+### Refactor
+
+- Remove unused or useless code paths in Secure-Join ([#4897](https://github.com/deltachat/deltachat-core-rust/pull/4897)).
+- Improve error handling in Secure-Join code.
+- Add hostname to "no DNS resolution results" error message.
+- Accept `&str` instead of `Option<String>` in idle().
+
+## [1.128.0] - 2023-11-02
+
+### Build system
+- [**breaking**] Upgrade nodejs version to 18 ([#4903](https://github.com/deltachat/deltachat-core-rust/pull/4903)).
+
+### Features / Changes
+
+- deltachat-rpc-client: Add `Account.wait_for_incoming_msg_event()`.
+- Decrease ratelimit for .testrun.org subdomains.
+
+### Fixes
+
+- Do not fail securejoin due to unrelated pending bobstate  ([#4896](https://github.com/deltachat/deltachat-core-rust/pull/4896)).
+- Allow other verified group recipients to be unverified, only check the sender verification.
+- Remove not working attempt to recover from verified key changes.
+
+## [1.127.2] - 2023-10-29
+
+### API-Changes
+
+- [**breaking**] Jsonrpc `misc_set_draft` now requires setting the viewtype.
+- jsonrpc: Add `get_message_info_object`.
+
+### Tests
+
+- deltachat-rpc-client: Move pytest option from pyproject.toml to tox.ini and set log level.
+- deltachat-rpc-client: Test securejoin.
+- Increase pytest timeout to 10 minutes.
+- Compile deltachat-rpc-server in debug mode for tests.
+
+## [1.127.1] - 2023-10-27
+
+### API-Changes
+
+- jsonrpc: add `.is_protection_broken` to `FullChat` and `BasicChat`.
+- jsonrpc: Add `id` to `ProviderInfo`.
+
+## [1.127.0] - 2023-10-26
+
+### API-Changes
+
+- [**breaking**] `dc_accounts_new` API is changed. Unused `os_name` argument is removed and `writable` argument is added.
+- jsonrpc: Add `resend_messages`.
+- [**breaking**] Remove unused function `is_verified_ex()` ([#4551](https://github.com/deltachat/deltachat-core-rust/pull/4551))
+- [**breaking**] Make `MsgId.delete_from_db()` private.
+- [**breaking**] deltachat-jsonrpc: use `kind` as a tag for all union types
+- json-rpc: Force stickers to be sent as stickers ([#4819](https://github.com/deltachat/deltachat-core-rust/pull/4819)).
+- Add mailto parse api ([#4829](https://github.com/deltachat/deltachat-core-rust/pull/4829)).
+- [**breaking**] Remove unused `DC_STR_PROTECTION_(EN)ABLED` strings
+- [**breaking**] Remove unused `dc_set_chat_protection()`
+- Hide `DcSecretKey` trait from the API.
+- Verified 1:1 chats ([#4315](https://github.com/deltachat/deltachat-core-rust/pull/4315)). Disabled by default, enable with `verified_one_on_one_chats` config.
+- Add api `chat::Chat::is_protection_broken`
+- Add `dc_chat_is_protection_broken()` C API.
+
+### CI
+
+- Run Rust tests with `RUST_BACKTRACE` set.
+- Replace `master` branch with `main`.  Run CI only on `main` branch pushes.
+- Test `deltachat-rpc-client` on Windows.
+
+### Documentation
+
+- Document how logs and error messages should be formatted in `CONTRIBUTING.md`.
+- Clarify transitive behaviour of `dc_contact_is_verfified()`.
+- Document `configured_addr`.
+
+### Features / Changes
+
+- Add lockfile to account manager ([#4314](https://github.com/deltachat/deltachat-core-rust/pull/4314)). 
+- Don't show a contact as verified if their key changed since the verification ([#4574](https://github.com/deltachat/deltachat-core-rust/pull/4574)).
+- deltachat-rpc-server: Add `--openrpc` option to print OpenRPC specification for JSON-RPC API. This specification can be used to generate JSON-RPC API clients.
+- Track whether contact is a bot or not ([#4821](https://github.com/deltachat/deltachat-core-rust/pull/4821)).
+- Replace `Config::SendSyncMsgs` with `SyncMsgs` ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
+
+### Fixes
+
+- Don't create 1:1 chat as protected for contact who doesn't prefer to encrypt ([#4538](https://github.com/deltachat/deltachat-core-rust/pull/4538)).
+- Allow to save a draft if the verification is broken ([#4542](https://github.com/deltachat/deltachat-core-rust/pull/4542)).
+- Fix info-message orderings of verified 1:1 chats ([#4545](https://github.com/deltachat/deltachat-core-rust/pull/4545)).
+- Fix example; this was changed some time ago, see https://docs.webxdc.org/spec.html#sendupdate
+- `receive_imf`: Update peerstate from db after handling Securejoin handshake ([#4600](https://github.com/deltachat/deltachat-core-rust/pull/4600)).
+- Sort old incoming messages below all outgoing ones ([#4621](https://github.com/deltachat/deltachat-core-rust/pull/4621)).
+- Do not mark non-verified group chats as verified when using securejoin.
+- `receive_imf`: Set protection only for Chattype::Single ([#4597](https://github.com/deltachat/deltachat-core-rust/pull/4597)).
+- Return from `dc_get_chatlist(DC_GCL_FOR_FORWARDING)` only chats where we can send ([#4616](https://github.com/deltachat/deltachat-core-rust/pull/4616)).
+- Clear VerifiedOneOnOneChats config on backup ([#4615](https://github.com/deltachat/deltachat-core-rust/pull/4615)).
+- Try removal of accounts multiple times with timeouts in case the database file is blocked (restore `try_many_times` workaround).
+
+### Build system
+
+- Remove examples/simple.rs.
+- Increase MSRV to 1.70.0.
+- Update dependencies.
+- Switch to iroh 0.4.x fork with updated dependencies.
+
+## [1.126.1] - 2023-10-24
+
+### Fixes
+
+- Do not hardcode version in deltachat-rpc-server source package.
+- Do not interrupt IMAP loop from `get_connectivity_html()`.
+
+### Features / Changes
+
+- imap: Buffer `STARTTLS` command.
+
+### Build system
+
+- Build `deltachat-rpc-server` binary for aarch64 macOS.
+- Build `deltachat-rpc-server` wheels for macOS and Windows.
+
+### Refactor
+
+- Remove job queue.
+
+### Miscellaneous Tasks
+
+- cargo: Update `ahash` to make `cargo-deny` happy.
+
+## [1.126.0] - 2023-10-22
+
+### API-Changes
+
+- Allow to filter by unread in `chatlist:try_load` ([#4824](https://github.com/deltachat/deltachat-core-rust/pull/4824)).
+- Add `misc_send_draft()` to JSON-RPC API ([#4839](https://github.com/deltachat/deltachat-core-rust/pull/4839)).
+
+### Features / Changes
+
+- [**breaking**] Make broadcast lists create their own chat ([#4644](https://github.com/deltachat/deltachat-core-rust/pull/4644)).
+  - This means that UIs need to ask for the name when creating a broadcast list, similar to <https://github.com/deltachat/deltachat-android/pull/2653>.
+- Add self-address to backup filename ([#4820](https://github.com/deltachat/deltachat-core-rust/pull/4820))
+
+### CI
+
+- Build Python wheels for deltachat-rpc-server.
+
+### Build system
+
+- Strip release binaries.
+- Workaround OpenSSL crate expecting libatomic to be available.
+
+### Fixes
+
+- Set `soft_heap_limit` on SQLite database.
+- imap: Fallback to `STATUS` if `SELECT` did not return UIDNEXT.
+
+## [1.125.0] - 2023-10-14
+
+### API-Changes
+
+- [**breaking**] deltachat-rpc-client: Replace `asyncio` with threads.
+- Validate boolean values passed to `set_config`. Attempts to set values other than `0` and `1` will result in an error.
+
+### CI
+
+- Reduce required Python version for deltachat-rpc-client from 3.8 to 3.7.
+
+### Features / Changes
+
+- Add developer option to disable IDLE.
+
+### Fixes
+
+- `deltachat-rpc-client`: Run `deltachat-rpc-server` in its own process group. This prevents reception of `SIGINT` by the server when the bot is terminated with `^C`.
+- python: Don't automatically set the displayname to "bot" when setting log level.
+- Don't update `timestamp`, `timestamp_rcvd`, `state` when replacing partially downloaded message ([#4700](https://github.com/deltachat/deltachat-core-rust/pull/4700)).
+- Assign encrypted partially downloaded group messages to 1:1 chat ([#4757](https://github.com/deltachat/deltachat-core-rust/pull/4757)).
+- Return all contacts from `Contact::get_all` for bots ([#4811](https://github.com/deltachat/deltachat-core-rust/pull/4811)).
+- Set connectivity status to "connected" during fake idle.
+- Return verifier contacts regardless of their origin.
+- Don't try to send more MDNs if there's a temporary SMTP error ([#4534](https://github.com/deltachat/deltachat-core-rust/pull/4534)).
+
+### Refactor
+
+- deltachat-rpc-client: Close stdin instead of sending `SIGTERM`.
+- deltachat-rpc-client: Remove print() calls. Standard `logging` package is for logging instead.
+
+### Tests
+
+- deltachat-rpc-client: Enable logs in pytest.
+
 ## [1.124.1] - 2023-10-05
 
 ### Fixes
@@ -928,7 +1469,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 ### Changes
 - Look at Authentication-Results. Don't accept Autocrypt key changes
-  if they come with negative authentiation results while this contact
+  if they come with negative authentication results while this contact
   sent emails with positive authentication results in the past. #3583
 - jsonrpc in cffi also sends events now #3662
 - jsonrpc: new format for events and better typescript autocompletion
@@ -2514,7 +3055,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 - delete all consumed secure-join handshake messagess #1209 #1212
 
-- rust-level cleanups #1218 #1217 #1210 #1205
+- Rust-level cleanups #1218 #1217 #1210 #1205
 
 - python-level cleanups #1204 #1202 #1201
 
@@ -2879,3 +3420,28 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0
 [1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0
 [1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1
+[1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0
+[1.126.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.125.0...v1.126.0
+[1.126.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.0...v1.126.1
+[1.127.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.1...v1.127.0
+[1.127.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.0...v1.127.1
+[1.127.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.1...v1.127.2
+[1.128.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.2...v1.128.0
+[1.129.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.128.0...v1.129.0
+[1.129.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.0...v1.129.1
+[1.130.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.1...v1.130.0
+[1.131.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.130.0...v1.131.0
+[1.131.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.0...v1.131.1
+[1.131.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.1...v1.131.2
+[1.131.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.2...v1.131.3
+[1.131.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.3...v1.131.4
+[1.131.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.4...v1.131.5
+[1.131.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.5...v1.131.6
+[1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
+[1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
+[1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
+[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
+[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1
+[1.133.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.1...v1.133.0
+[1.133.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.0...v1.133.1
+[1.133.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.1...v1.133.2

CONTRIBUTING.md

@@ -76,6 +76,40 @@ If you have multiple changes in one PR, create multiple conventional commits, an
 [Conventional Commits]: https://www.conventionalcommits.org/
 [git-cliff]: https://git-cliff.org/
 
+### Errors
+
+Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.
+When using [`Context`](https://docs.rs/anyhow/latest/anyhow/trait.Context.html),
+capitalize it but do not add a full stop as the contexts will be separated by `:`.
+For example:
+```
+.with_context(|| format!("Unable to trash message {msg_id}"))
+```
+
+All errors should be handled in one of these ways:
+- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+- With `.log_err().ok()`.
+- Bubbled up with `?`.
+
+`backtrace` feature is enabled for `anyhow` crate
+and `debug = 1` option is set in the test profile.
+This allows to run `RUST_BACKTRACE=1 cargo test`
+and get a backtrace with line numbers in resultified tests
+which return `anyhow::Result`.
+
+### Logging
+
+For logging, use `info!`, `warn!` and `error!` macros.
+Log messages should be capitalized and have a full stop in the end. For example:
+```
+info!(context, "Ignoring addition of {added_addr:?} to {chat_id}.");
+```
+
+Format anyhow errors with `{:#}` to print all the contexts like this:
+```
+error!(context, "Failed to set selfavatar timestamp: {err:#}.");
+```
+
 ### Reviewing
 
 Once a PR has an approval and passes CI, it can be merged.

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.124.1"
+version = "1.133.2"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.65"
+rust-version = "1.70"
 
 [profile.dev]
 debug = 0
@@ -11,6 +11,10 @@ panic = 'abort'
 opt-level = 1
 
 [profile.test]
+# Make anyhow `backtrace` feature useful.
+# With `debug = 0` there are no line numbers in the backtrace
+# produced with RUST_BACKTRACE=1.
+debug = 1
 opt-level = 0
 
 # Always optimize dependencies.
@@ -24,10 +28,10 @@ lto = true
 panic = 'abort'
 opt-level = "z"
 codegen-units = 1
+strip = true
 
 [patch.crates-io]
-quinn-udp = { git = "https://github.com/quinn-rs/quinn", branch="main" }
-quinn-proto = { git = "https://github.com/quinn-rs/quinn", branch="main" }
+imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
@@ -35,25 +39,27 @@ format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
-async-channel = "1.8.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
+async-channel = "2.0.0"
+async-imap = { version = "0.9.5", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
 backtrace = "0.3"
 base64 = "0.21"
-brotli = { version = "3.3", default-features=false, features = ["std"] }
+brotli = { version = "3.4", default-features=false, features = ["std"] }
 chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
-fast-socks5 = "0.8"
+fast-socks5 = "0.9"
+fd-lock = "4"
 futures = "0.3"
-futures-lite = "1.13.0"
+futures-lite = "2.0.0"
 hex = "0.4.0"
+hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.6", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh = { version = "0.4.1", default-features = false }
+image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -66,16 +72,18 @@ once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
 pgp = { version = "0.10", default-features = false }
+pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
-quick-xml = "0.29"
+quick-xml = "0.31"
+quoted_printable = "0.5"
 rand = "0.8"
-regex = "1.8"
-reqwest = { version = "0.11.18", features = ["json"] }
-rusqlite = { version = "0.29", features = ["sqlcipher"] }
+regex = "1.9"
+reqwest = { version = "0.11.23", features = ["json"] }
+rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-sanitize-filename = "0.4"
-serde_json = "1.0"
+sanitize-filename = "0.5"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
@@ -89,21 +97,29 @@ tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
-tokio-util = "0.7.8"
-toml = "0.7"
-trust-dns-resolver = "0.22"
+tokio-util = "0.7.9"
+toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
+# Pin OpenSSL to 3.1 releases.
+# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
+# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
+# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
+# According to <https://www.openssl.org/policies/releasestrat.html>
+# 3.1 branch will be supported until 2025-03-14.
+openssl-src = "~300.1"
+
 [dev-dependencies]
 ansi_term = "0.12.0"
+anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "1.13"
+futures-lite = "2.0.0"
 log = "0.4"
 pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
-testdir = "0.8.0"
+testdir = "0.9.0"
 tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 pretty_assertions = "1.3.0"
 
@@ -118,11 +134,6 @@ members = [
   "format-flowed",
 ]
 
-[[example]]
-name = "simple"
-path = "examples/simple.rs"
-
-
 [[bench]]
 name = "create_account"
 harness = false

README.md

@@ -1,8 +1,16 @@
-# Delta Chat Rust
+<p align="center">
+  <img alt="Delta Chat Logo" height="200px" src="https://raw.githubusercontent.com/deltachat/deltachat-pages/master/assets/blog/rust-delta.png">
+</p>
 
-> Deltachat-core written in Rust 
+<p align="center">
+  <a href="https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml">
+    <img alt="Rust CI" src="https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg">
+  </a>
+</p>
 
-[![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+<p align="center">
+The core library for Delta Chat, written in Rust
+</p>
 
 ## Installing Rust and Cargo
 
@@ -19,7 +27,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -113,7 +121,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests

benches/create_account.rs

@@ -8,7 +8,8 @@ async fn create_accounts(n: u32) {
     let dir = tempdir().unwrap();
     let p: PathBuf = dir.path().join("accounts");
 
-    let mut accounts = Accounts::new(p.clone()).await.unwrap();
+    let writable = true;
+    let mut accounts = Accounts::new(p.clone(), writable).await.unwrap();
 
     for expected_id in 2..n {
         let id = accounts.add_account().await.unwrap();

cliff.toml

@@ -54,7 +54,7 @@ header = """
 # Changelog\n
 """
 # template for the changelog body
-# https://tera.netlify.app/docs/#introduction
+# https://keats.github.io/tera/docs/#templates
 body = """
 {% if version %}\
     ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.124.1"
+version = "1.133.2"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/Doxyfile

@@ -846,7 +846,7 @@ EXCLUDE_PATTERNS       =
 # exclude all test directories use the pattern */test/*
 
 ######################################################
-EXCLUDE_SYMBOLS        = dc_aheader_t dc_apeerstate_t dc_e2ee_helper_t dc_imap_t dc_job*_t dc_key_t dc_keyring_t dc_loginparam_t dc_mime*_t
+EXCLUDE_SYMBOLS        = dc_aheader_t dc_apeerstate_t dc_e2ee_helper_t dc_imap_t dc_job*_t dc_key_t dc_loginparam_t dc_mime*_t
 EXCLUDE_SYMBOLS       += dc_saxparser_t dc_simplify_t dc_smtp_t dc_sqlite3_t dc_strbuilder_t dc_param_t dc_hash_t dc_hashelem_t
 EXCLUDE_SYMBOLS       += _dc_* jsmn*
 ######################################################

deltachat-ffi/DoxygenLayout.xml

@@ -9,7 +9,7 @@
       <tab type="hierarchy" visible="no" title="" intro=""/>
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="topics" visible="yes" title="Constants" intro="Here is a list of constants:"/>
     <tab type="pages" visible="yes" title="" intro=""/>
     <tab type="namespaces" visible="yes" title="">
       <tab type="namespacelist" visible="yes" title="" intro=""/>

deltachat-ffi/deltachat.h

@@ -25,7 +25,6 @@ typedef struct _dc_event     dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 typedef struct _dc_backup_provider dc_backup_provider_t;
-typedef struct _dc_http_response dc_http_response_t;
 
 // Alias for backwards compatibility, use dc_event_emitter_t instead.
 typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
@@ -384,7 +383,12 @@ char*           dc_get_blobdir               (const dc_context_t* context);
 /**
  * Configure the context. The configuration is handled by key=value pairs as:
  *
- * - `addr`         = address to display (always needed)
+ * - `addr`         = Email address to use for configuration.
+ *                    If dc_configure() fails this is not the email address actually in use.
+ *                    Use `configured_addr` to find out the email address actually in use.
+ * - `configured_addr` = Email address actually in use.
+ *                    Unless for testing, do not set this value using dc_set_config().
+ *                    Instead, set `addr` and call dc_configure().
  * - `mail_server`  = IMAP-server, guessed if left out
  * - `mail_user`    = IMAP-username, guessed if left out
  * - `mail_pw`      = IMAP-password (always needed)
@@ -492,6 +496,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
+ *                    0=use IMAP IDLE if the server supports it.
+ *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
  *                    For larger messages, only the header is downloaded and a placeholder is shown.
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
@@ -500,6 +507,16 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
+ * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
+ *                    seconds. 2 days by default.
+ *                    This is not supposed to be changed by UIs and only used for testing.
+ * - `verified_one_on_one_chats` = Feature flag for verified 1:1 chats; the UI should set it
+ *                    to 1 if it supports verified 1:1 chats.
+ *                    Regardless of this setting, `dc_chat_is_protected()` returns true while the key is verified,
+ *                    and when the key changes, an info message is posted into the chat.
+ *                    0=Nothing else happens when the key changes.
+ *                    1=After the key changed, `dc_chat_can_send()` returns false and `dc_chat_is_protection_broken()` returns true
+ *                    until `dc_accept_chat()` is called.
  * - `ui.*`         = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
  *                    The prefix should be followed by the system and maybe subsystem,
  *                    e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
@@ -878,7 +895,8 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *     - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
  *       is added as needed.
  * @param query_str An optional query for filtering the list. Only chats matching this query
- *     are returned. Give NULL for no filtering.
+ *     are returned. Give NULL for no filtering. When `is:unread` is contained in the query,
+ *     the chatlist is filtered such that only chats with unread messages show up.
  * @param query_id An optional contact ID for filtering the list. Only chats including this contact ID
  *     are returned. Give 0 for no filtering.
  * @return A chatlist as an dc_chatlist_t object.
@@ -1093,6 +1111,7 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * received overrides all previously received reactions. It is
  * possible to remove all reactions by sending an empty string.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `send_reaction` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id ID of the message you react to.
@@ -1105,6 +1124,7 @@ uint32_t dc_send_reaction (dc_context_t* context, uint32_t msg_id, char *reactio
 /**
  * Get a structure with reactions to the message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id The message ID to get reactions for.
@@ -1118,7 +1138,7 @@ dc_reactions_t* dc_get_msg_reactions (dc_context_t *context, int msg_id);
  *
  * In JS land, that would be mapped to something as:
  * ```
- * success = window.webxdc.sendUpdate('{"action":"move","src":"A3","dest":"B4"}', 'move A3 B4');
+ * success = window.webxdc.sendUpdate('{payload: {"action":"move","src":"A3","dest":"B4"}}', 'move A3 B4');
  * ```
  * `context` and `msg_id` are not needed in JS as those are unique within a webxdc instance.
  * See dc_get_webxdc_status_updates() for the receiving counterpart.
@@ -1500,24 +1520,6 @@ dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t ch
 uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t msg_id, int dir, int msg_type, int msg_type2, int msg_type3);
 
 
-/**
- * Enable or disable protection against active attacks.
- * To enable protection, it is needed that all members are verified;
- * if this condition is met, end-to-end-encryption is always enabled
- * and only the verified keys are used.
- *
- * Sends out #DC_EVENT_CHAT_MODIFIED on changes
- * and #DC_EVENT_MSGS_CHANGED if a status message was sent.
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param chat_id The ID of the chat to change the protection for.
- * @param protect 1=protect chat, 0=unprotect chat
- * @return 1=success, 0=error, e.g. some members may be unverified
- */
-int             dc_set_chat_protection       (dc_context_t* context, uint32_t chat_id, int protect);
-
-
 /**
  * Set chat visibility to pinned, archived or normal.
  *
@@ -1711,24 +1713,12 @@ uint32_t        dc_create_group_chat         (dc_context_t* context, int protect
  * Create a new broadcast list.
  *
  * Broadcast lists are similar to groups on the sending device,
- * however, recipients get the messages in normal one-to-one chats
- * and will not be aware of other members.
+ * however, recipients get the messages in a read-only chat
+ * and will see who the other members are.
  *
- * Replies to broadcasts go only to the sender
- * and not to all broadcast recipients.
- * Moreover, replies will not appear in the broadcast list
- * but in the one-to-one chat with the person answering.
- *
- * The name and the image of the broadcast list is set automatically
- * and is visible to the sender only.
- * Not asking for these data allows more focused creation
- * and we bypass the question who will get which data.
- * Also, many users will have at most one broadcast list
- * so, a generic name and image is sufficient at the first place.
- *
- * Later on, however, the name can be changed using dc_set_chat_name().
- * The image cannot be changed to have a unique, recognizable icon in the chat lists.
- * All in all, this is also what other messengers are doing here.
+ * For historical reasons, this function does not take a name directly,
+ * instead you have to set the name using dc_set_chat_name()
+ * after creating the broadcast list.
  *
  * @memberof dc_context_t
  * @param context The context object.
@@ -2271,8 +2261,7 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  *   the backup is not encrypted.
  *   The backup contains all contacts, chats, images and other data and device independent settings.
  *   The backup does not contain device dependent settings as ringtones or LED notification settings.
- *   The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
- *   the format is `delta-chat-<day>-<number>.tar`
+ *   The name of the backup is `delta-chat-backup-<day>-<number>-<addr>.tar`.
  *
  * - **DC_IMEX_IMPORT_BACKUP** (12) - `param1` is the file (not: directory) to import. `param2` is the passphrase.
  *   The file is normally created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
@@ -2573,7 +2562,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
- *     See https://countermitm.readthedocs.io/en/latest/new.html
+ *     See https://securejoin.readthedocs.io/en/latest/new.html
  *     for details about both protocols.
  * @return The text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
@@ -2609,7 +2598,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  *
  * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
- * See https://countermitm.readthedocs.io/en/latest/new.html
+ * See https://securejoin.readthedocs.io/en/latest/new.html
  * for details about both protocols.
  *
  * @memberof dc_context_t
@@ -2953,16 +2942,18 @@ int dc_receive_backup (dc_context_t* context, const char* qr);
  * use dc_accounts_remove_account().
  *
  * @memberof dc_accounts_t
- * @param os_name
  * @param dir The directory to create the context-databases in.
  *     If the directory does not exist,
  *     dc_accounts_new() will try to create it.
+ * @param writable Whether the returned account manager is writable, i.e. calling these functions on
+ *     it is possible: dc_accounts_add_account(), dc_accounts_add_closed_account(),
+ *     dc_accounts_migrate_account(), dc_accounts_remove_account(), dc_accounts_select_account().
  * @return An account manager object.
  *     The object must be passed to the other account manager functions
  *     and must be freed using dc_accounts_unref() after usage.
  *     On errors, NULL is returned.
  */
-dc_accounts_t* dc_accounts_new                  (const char* os_name, const char* dir);
+dc_accounts_t* dc_accounts_new                  (const char* dir, int writable);
 
 
 /**
@@ -3741,9 +3732,22 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 
 /**
  * Check if a chat is protected.
- * Protected chats contain only verified members and encryption is always enabled.
- * Protected chats are created using dc_create_group_chat() by setting the 'protect' parameter to 1.
- * The status can be changed using dc_set_chat_protection().
+ *
+ * End-to-end encryption is guaranteed in protected chats
+ * and only verified contacts
+ * as determined by dc_contact_is_verified()
+ * can be added to protected chats.
+ *
+ * Protected chats are created using dc_create_group_chat()
+ * by setting the 'protect' parameter to 1.
+ * 1:1 chats become protected or unprotected automatically
+ * if `verified_one_on_one_chats` setting is enabled.
+ *
+ * UI should display a green checkmark
+ * in the chat title,
+ * in the chatlist item
+ * and in the chat profile
+ * if chat protection is enabled.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -3752,6 +3756,26 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 int             dc_chat_is_protected         (const dc_chat_t* chat);
 
 
+/**
+ * Checks if the chat was protected, and then an incoming message broke this protection.
+ *
+ * This function is only useful if the UI enabled the `verified_one_on_one_chats` feature flag,
+ * otherwise it will return false for all chats.
+ *
+ * 1:1 chats are automatically set as protected when a contact is verified.
+ * When a message comes in that is not encrypted / signed correctly,
+ * the chat is automatically set as unprotected again.
+ * dc_chat_is_protection_broken() will return true until dc_accept_chat() is called.
+ *
+ * The UI should let the user confirm that this is OK with a message like
+ * `Bob sent a message from another device. Tap to learn more` and then call dc_accept_chat().
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat protection broken, 0=otherwise.
+ */
+int             dc_chat_is_protection_broken (const dc_chat_t* chat);
+
+
 /**
  * Check if locations are sent to the chat
  * at the time the object was created using dc_get_chat().
@@ -3956,7 +3980,7 @@ int64_t          dc_msg_get_received_timestamp (const dc_msg_t* msg);
  * Get the message time used for sorting.
  * This function returns the timestamp that is used for sorting the message
  * into lists as returned e.g. by dc_get_chat_msgs().
- * This may be the reveived time, the sending time or another time.
+ * This may be the received time, the sending time or another time.
  *
  * To get the receiving time, use dc_msg_get_received_timestamp().
  * To get the sending time, use dc_msg_get_timestamp().
@@ -4346,7 +4370,7 @@ int             dc_msg_is_forwarded           (const dc_msg_t* msg);
  * Check if the message is an informational message, created by the
  * device or by another users. Such messages are not "typed" by the user but
  * created due to other actions,
- * e.g. dc_set_chat_name(), dc_set_chat_profile_image(), dc_set_chat_protection()
+ * e.g. dc_set_chat_name(), dc_set_chat_profile_image(),
  * or dc_add_contact_to_chat().
  *
  * These messages are typically shown in the center of the chat view,
@@ -4373,6 +4397,9 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * Currently, the following types are defined:
  * - DC_INFO_PROTECTION_ENABLED (11) - Info-message for "Chat is now protected"
  * - DC_INFO_PROTECTION_DISABLED (12) - Info-message for "Chat is no longer protected"
+ * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
+ *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
+ *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4399,6 +4426,7 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_EPHEMERAL_TIMER_CHANGED   10
 #define         DC_INFO_PROTECTION_ENABLED        11
 #define         DC_INFO_PROTECTION_DISABLED       12
+#define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
 /**
@@ -4557,15 +4585,18 @@ int dc_msg_has_html (dc_msg_t* msg);
   * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
   *
   * The function returns one of:
-  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
-  *                                  and should be rendered as usual.
-  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
-  *                                  In addition to the usual message rendering,
-  *                                  the UI shall show a download button that calls dc_download_full_msg()
-  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
-  *                                  If the download fails or succeeds,
-  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
-  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  * - @ref DC_DOWNLOAD_DONE           - The message does not need any further download action
+  *                                     and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE      - There is additional content to download.
+  *                                     In addition to the usual message rendering,
+  *                                     the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS    - Download was started with dc_download_full_msg() and is still in progress.
+  *                                     If the download fails or succeeds,
+  *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  *
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  *                                     It was fully downloaded, but we failed to decrypt it.
+  * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
   * @memberof dc_msg_t
   * @param msg The message object.
@@ -5023,10 +5054,16 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 
 
 /**
- * Check if a contact was verified. E.g. by a secure-join QR code scan
- * and if the key has not changed since this verification.
+ * Check if the contact
+ * can be added to verified chats,
+ * i.e. has a verified key
+ * and Autocrypt key matches the verified key.
  *
- * The UI may draw a checkbox or something like that beside verified contacts.
+ * If contact is verified
+ * UI should display green checkmark after the contact name
+ * in contact list items,
+ * in chat member list items
+ * and in profiles if no chat with the contact exist (otherwise, use dc_chat_is_protected()).
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5035,28 +5072,29 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
  */
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
-
-
 /**
- * Return the address that verified a contact
- *
- * The UI may use this in addition to a checkmark showing the verification status
+ * Returns whether contact is a bot.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return 
- *    A string containing the verifiers address. If it is the same address as the contact itself,
- *    we verified the contact ourself. If it is an empty string, we don't have verifier 
- *    information or the contact is not verified.
- * @deprecated 2023-09-28, use dc_contact_get_verifier_id instead
+ * @return 0 if the contact is not a bot, 1 otherwise.
  */
-char*           dc_contact_get_verifier_addr       (dc_contact_t* contact);
+int             dc_contact_is_bot            (dc_contact_t* contact);
 
 
 /**
- * Return the `ContactId` that verified a contact
+ * Return the contact ID that verified a contact.
  *
- * The UI may use this in addition to a checkmark showing the verification status
+ * If the function returns non-zero result,
+ * display green checkmark in the profile and "Introduced by ..." line
+ * with the name and address of the contact
+ * formatted by dc_contact_get_name_n_addr.
+ *
+ * If this function returns a verifier,
+ * this does not necessarily mean
+ * you can add the contact to verified chats.
+ * Use dc_contact_is_verified() to check
+ * if a contact can be added to a verified chat instead.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5161,72 +5199,6 @@ int             dc_provider_get_status                (const dc_provider_t* prov
 void            dc_provider_unref                     (dc_provider_t* provider);
 
 
-/**
- * Return an HTTP(S) GET response.
- * This function can be used to download remote content for HTML emails.
- *
- * @memberof dc_context_t
- * @param context The context object to take proxy settings from.
- * @param url HTTP or HTTPS URL.
- * @return The response must be released using dc_http_response_unref() after usage.
- *     NULL is returned on errors.
- */
-dc_http_response_t*     dc_get_http_response      (const dc_context_t* context, const char* url);
-
-
-/**
- * @class dc_http_response_t
- *
- * An object containing an HTTP(S) GET response.
- * Created by dc_get_http_response().
- */
-
-
-/**
- * Returns HTTP response MIME type as a string, e.g. "text/plain" or "text/html".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_mimetype (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response encoding, e.g. "utf-8".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_encoding (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response contents.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob which must be released using dc_str_unref() after usage. NULL is never returned.
- */
-uint8_t*                dc_http_response_get_blob     (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response content size.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob size.
- */
-size_t                  dc_http_response_get_size     (const dc_http_response_t* response);
-
-/**
- * Free an HTTP response object.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- */
-void                    dc_http_response_unref        (const dc_http_response_t* response);
-
-
 /**
  * @class dc_lot_t
  *
@@ -5326,6 +5298,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * @class dc_reactions_t
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  *
  * An object representing all reactions for a single message.
  */
@@ -5333,6 +5306,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * Returns array of contacts which reacted to the given message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @return array of contact IDs. Use dc_array_get_cnt() to get array length and
@@ -5344,6 +5318,7 @@ dc_array_t*     dc_reactions_get_contacts(dc_reactions_t* reactions);
 /**
  * Returns a string containing space-separated reactions of a single contact.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @param contact_id ID of the contact.
@@ -5359,6 +5334,7 @@ char*           dc_reactions_get_by_contact_id(dc_reactions_t* reactions, uint32
  *
  * Reactions objects are created by dc_get_msg_reactions().
  *
+ * @deprecated 2023-11-27
  * @memberof dc_reactions_t
  * @param reactions The object to free.
  *     If NULL is given, nothing is done.
@@ -6219,6 +6195,7 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ *     1000=vg-member-added/vc-contact-confirm received
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
@@ -6242,6 +6219,18 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_SELFAVATAR_CHANGED               2110
 
 
+/**
+ * A multi-device synced config value changed. Maybe the app needs to refresh smth. For uniformity
+ * this is emitted on the source device too. The value isn't reported, otherwise it would be logged
+ * which might not be good for privacy. You can get the new value with
+ * `dc_get_config(context, data2)`.
+ *
+ * @param data1 0
+ * @param data2 (char*) Configuration key.
+ */
+#define DC_EVENT_CONFIG_SYNCED                    2111
+
+
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
@@ -6411,22 +6400,27 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Download not needed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_DONE          0
+#define DC_DOWNLOAD_DONE           0
 
 /**
  * Download available, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_AVAILABLE    10
+#define DC_DOWNLOAD_AVAILABLE      10
 
 /**
  * Download failed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_FAILURE      20
+#define DC_DOWNLOAD_FAILURE        20
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_UNDECIPHERABLE 30
 
 /**
  * Download in progress, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_IN_PROGRESS  1000
+#define DC_DOWNLOAD_IN_PROGRESS    1000
 
 
 
@@ -6591,7 +6585,7 @@ void dc_event_unref(dc_event_t* event);
 /// - %1$s will be replaced by the name of the verified contact
 #define DC_STR_CONTACT_VERIFIED           35
 
-/// "Cannot verify %1$s."
+/// "Cannot establish guaranteed end-to-end encryption with %1$s."
 ///
 /// Used in status messages.
 /// - %1$s will be replaced by the name of the contact that cannot be verified
@@ -6781,15 +6775,6 @@ void dc_event_unref(dc_event_t* event);
 /// Used in error strings.
 #define DC_STR_ERROR_NO_NETWORK           87
 
-/// "Chat protection enabled."
-///
-
-/// @deprecated Deprecated, replaced by DC_STR_MSG_YOU_ENABLED_PROTECTION and DC_STR_MSG_PROTECTION_ENABLED_BY.
-#define DC_STR_PROTECTION_ENABLED         88
-
-/// @deprecated Deprecated, replaced by DC_STR_MSG_YOU_DISABLED_PROTECTION and DC_STR_MSG_PROTECTION_DISABLED_BY.
-#define DC_STR_PROTECTION_DISABLED        89
-
 /// "Reply"
 ///
 /// Used in summaries.
@@ -7038,6 +7023,8 @@ void dc_event_unref(dc_event_t* event);
 /// "You added member %1$s."
 ///
 /// Used in status messages.
+///
+/// `%1$s` will be replaced by the added member's name.
 #define DC_STR_ADD_MEMBER_BY_YOU 128
 
 /// "Member %1$s added by %2$s."
@@ -7234,26 +7221,6 @@ void dc_event_unref(dc_event_t* event);
 /// `%2$s` will be replaced by name and address of the contact.
 #define DC_STR_EPHEMERAL_TIMER_WEEKS_BY_OTHER 157
 
-/// "You enabled chat protection."
-///
-/// Used in status messages.
-#define DC_STR_PROTECTION_ENABLED_BY_YOU 158
-
-/// "Chat protection enabled by %1$s."
-///
-/// `%1$s` will be replaced by name and address of the contact.
-///
-/// Used in status messages.
-#define DC_STR_PROTECTION_ENABLED_BY_OTHER 159
-
-/// "You disabled chat protection."
-#define DC_STR_PROTECTION_DISABLED_BY_YOU 160
-
-/// "Chat protection disabled by %1$s."
-///
-/// `%1$s` will be replaced by name and address of the contact.
-#define DC_STR_PROTECTION_DISABLED_BY_OTHER 161
-
 /// "Scan to set up second device for %1$s"
 ///
 /// `%1$s` will be replaced by name and address of the account.
@@ -7264,6 +7231,36 @@ void dc_event_unref(dc_event_t* event);
 /// Used as a device message after a successful backup transfer.
 #define DC_STR_BACKUP_TRANSFER_MSG_BODY 163
 
+/// "Messages are guaranteed to be end-to-end encrypted from now on."
+///
+/// Used in info messages.
+#define DC_STR_CHAT_PROTECTION_ENABLED 170
+
+/// "%1$s sent a message from another device."
+///
+/// Used in info messages.
+#define DC_STR_CHAT_PROTECTION_DISABLED 171
+
+/// "Others will only see this group after you sent a first message."
+///
+/// Used as the first info messages in newly created groups.
+#define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
+
+/// "Member %1$s added."
+///
+/// Used as info messages.
+///
+/// `%1$s` will be replaced by the added member's name.
+#define DC_STR_MESSAGE_ADD_MEMBER 173
+
+/// "Your email provider %1$s requires end-to-end encryption which is not setup yet."
+///
+/// Used as info messages when a message cannot be sent because it cannot be encrypted.
+///
+/// `%1$s` will be replaced by the provider's domain.
+#define DC_STR_INVALID_UNENCRYPTED_MAIL 174
+
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -29,9 +29,8 @@ use deltachat::contact::{Contact, ContactId, Origin};
 use deltachat::context::Context;
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
-use deltachat::key::{DcKey, DcSecretKey};
+use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
-use deltachat::net::read_url_blob;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
@@ -489,7 +488,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     if context.is_null() {
         return;
     }
-    let ctx = &*context;
+    let ctx = &mut *context;
 
     block_on(ctx.start_io())
 }
@@ -557,6 +556,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SecurejoinJoinerProgress { .. } => 2061,
         EventType::ConnectivityChanged => 2100,
         EventType::SelfavatarChanged => 2110,
+        EventType::ConfigSynced { .. } => 2111,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
     }
@@ -584,6 +584,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Error(_)
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
@@ -644,7 +645,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::SelfavatarChanged => 0,
+        | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. } => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
@@ -723,6 +725,10 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             .to_c_string()
             .unwrap_or_default()
             .into_raw(),
+        EventType::ConfigSynced { key } => {
+            let data2 = key.to_string().to_c_string().unwrap_or_default();
+            data2.into_raw()
+        }
     }
 }
 
@@ -813,21 +819,12 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
         return 0;
     }
     let ctx = &*context;
-    block_on(async move {
-        let addr = tools::EmailAddress::new(&to_string_lossy(addr))?;
-        let secret = key::SignedSecretKey::from_asc(&to_string_lossy(secret_data))?.0;
-        let public = secret.split_public_key()?;
-        let keypair = key::KeyPair {
-            addr,
-            public,
-            secret,
-        };
-        key::store_self_keypair(ctx, &keypair, key::KeyPairUse::Default).await?;
-        Ok::<_, anyhow::Error>(1)
-    })
-    .context("Failed to save keypair")
-    .log_err(ctx)
-    .unwrap_or(0)
+    let addr = to_string_lossy(addr);
+    let secret_data = to_string_lossy(secret_data);
+    block_on(preconfigure_keypair(ctx, &addr, &secret_data))
+        .context("Failed to save keypair")
+        .log_err(ctx)
+        .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1472,32 +1469,6 @@ pub unsafe extern "C" fn dc_get_next_media(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_set_chat_protection(
-    context: *mut dc_context_t,
-    chat_id: u32,
-    protect: libc::c_int,
-) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_set_chat_protection()");
-        return 0;
-    }
-    let ctx = &*context;
-    let protect = if let Some(s) = ProtectionStatus::from_i32(protect) {
-        s
-    } else {
-        warn!(ctx, "bad protect-value for dc_set_chat_protection()");
-        return 0;
-    };
-
-    block_on(async move {
-        match ChatId::new(chat_id).set_protection(ctx, protect).await {
-            Ok(()) => 1,
-            Err(_) => 0,
-        }
-    })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
@@ -3132,6 +3103,16 @@ pub unsafe extern "C" fn dc_chat_is_protected(chat: *mut dc_chat_t) -> libc::c_i
     ffi_chat.chat.is_protected() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_is_protection_broken(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_is_protection_broken()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.is_protection_broken() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_is_sending_locations(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
@@ -4138,27 +4119,26 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(ffi_contact.contact.is_verified(ctx))
+    if block_on(ffi_contact.contact.is_verified(ctx))
         .context("is_verified failed")
         .log_err(ctx)
-        .unwrap_or_default() as libc::c_int
+        .unwrap_or_default()
+    {
+        // Return value is essentially a boolean,
+        // but we return 2 for true for backwards compatibility.
+        2
+    } else {
+        0
+    }
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_contact_get_verifier_addr(
-    contact: *mut dc_contact_t,
-) -> *mut libc::c_char {
+pub unsafe extern "C" fn dc_contact_is_bot(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
-        eprintln!("ignoring careless call to dc_contact_get_verifier_addr()");
-        return "".strdup();
+        eprintln!("ignoring careless call to dc_contact_is_bot()");
+        return 0;
     }
-    let ffi_contact = &*contact;
-    let ctx = &*ffi_contact.context;
-    block_on(ffi_contact.contact.get_verifier_addr(ctx))
-        .context("failed to get verifier for contact")
-        .log_err(ctx)
-        .unwrap_or_default()
-        .strdup()
+    (*contact).contact.is_bot() as libc::c_int
 }
 
 #[no_mangle]
@@ -4632,96 +4612,6 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // this may change once we start localizing string.
 }
 
-// dc_http_response_t
-
-pub type dc_http_response_t = net::HttpResponse;
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_http_response(
-    context: *const dc_context_t,
-    url: *const libc::c_char,
-) -> *mut dc_http_response_t {
-    if context.is_null() || url.is_null() {
-        eprintln!("ignoring careless call to dc_get_http_response()");
-        return ptr::null_mut();
-    }
-
-    let context = &*context;
-    let url = to_string_lossy(url);
-    if let Ok(response) = block_on(read_url_blob(context, &url))
-        .context("read_url_blob")
-        .log_err(context)
-    {
-        Box::into_raw(Box::new(response))
-    } else {
-        ptr::null_mut()
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_mimetype(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_mimetype()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.mimetype.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_encoding(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_encoding()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.encoding.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_blob(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_blob()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    let blob_len = response.blob.len();
-    let ptr = libc::malloc(blob_len);
-    libc::memcpy(ptr, response.blob.as_ptr() as *mut libc::c_void, blob_len);
-    ptr as *mut libc::c_char
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_size(
-    response: *const dc_http_response_t,
-) -> libc::size_t {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_size()");
-        return 0;
-    }
-
-    let response = &*response;
-    response.blob.len()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_unref(response: *mut dc_http_response_t) {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_unref()");
-        return;
-    }
-    drop(Box::from_raw(response));
-}
-
 // -- Accounts
 
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
@@ -4750,17 +4640,17 @@ pub type dc_accounts_t = AccountsWrapper;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_new(
-    _os_name: *const libc::c_char,
-    dbfile: *const libc::c_char,
+    dir: *const libc::c_char,
+    writable: libc::c_int,
 ) -> *mut dc_accounts_t {
     setup_panic!();
 
-    if dbfile.is_null() {
+    if dir.is_null() {
         eprintln!("ignoring careless call to dc_accounts_new()");
         return ptr::null_mut();
     }
 
-    let accs = block_on(Accounts::new(as_path(dbfile).into()));
+    let accs = block_on(Accounts::new(as_path(dir).into(), writable != 0));
 
     match accs {
         Ok(accs) => Box::into_raw(Box::new(AccountsWrapper::new(accs))),
@@ -4971,8 +4861,8 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
         return;
     }
 
-    let accounts = &*accounts;
-    block_on(async move { accounts.read().await.start_io().await });
+    let accounts = &mut *accounts;
+    block_on(async move { accounts.write().await.start_io().await });
 }
 
 #[no_mangle]

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.124.1"
+version = "1.133.2"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -15,26 +15,26 @@ required-features = ["webserver"]
 anyhow = "1"
 deltachat = { path = ".." }
 num-traits = "0.2"
-schemars = "0.8.11"
+schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.6.0"
+tempfile = "3.9.0"
 log = "0.4"
-async-channel = { version = "1.8.0" }
-futures = { version = "0.3.28" }
-serde_json = "1.0.99"
-yerpc = { version = "0.5.1", features = ["anyhow_expose", "openrpc"] }
-typescript-type-def = { version = "0.5.5", features = ["json_value"] }
-tokio = { version = "1.29.1" }
-sanitize-filename = "0.4"
+async-channel = { version = "2.0.0" }
+futures = { version = "0.3.30" }
+serde_json = "1"
+yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
+typescript-type-def = { version = "0.5.8", features = ["json_value"] }
+tokio = { version = "1.33.0" }
+sanitize-filename = "0.5"
 walkdir = "2.3.3"
 base64 = "0.21"
 
 # optional dependencies
-axum = { version = "0.6.18", optional = true, features = ["ws"] }
+axum = { version = "0.7", optional = true, features = ["ws"] }
 env_logger = { version = "0.10.0", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.29.1", features = ["full", "rt-multi-thread"] }
+tokio = { version = "1.33.0", features = ["full", "rt-multi-thread"] }
 
 
 [features]

deltachat-jsonrpc/README.md

@@ -108,10 +108,10 @@ This will build the `deltachat-jsonrpc-server` binary and then run a test suite
 
 The test suite includes some tests that need online connectivity and a way to create test email accounts. To run these tests, talk to DeltaChat developers to get a token for the `testrun.org` service, or use a local instance of [`mailadm`](https://github.com/deltachat/docker-mailadm).
 
-Then, set the `DCC_NEW_TMP_EMAIL` environment variable to your mailadm token before running the tests.
+Then, set the `CHATMAIL_DOMAIN` environment variable to your testing email server domain.
 
 ```
-DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=yourtoken npm run test
+CHATMAIL_DOMAIN=chat.example.org npm run test
 ```
 
 #### Test Coverage

deltachat-jsonrpc/src/api.rs

@@ -4,30 +4,30 @@ use std::{collections::HashMap, str::FromStr};
 
 use anyhow::{anyhow, bail, ensure, Context, Result};
 pub use deltachat::accounts::Accounts;
-use deltachat::message::get_msg_read_receipts;
-use deltachat::qr::Qr;
-use deltachat::{
-    chat::{
-        self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, get_chat_msgs_ex,
-        marknoticed_chat, remove_contact_from_chat, Chat, ChatId, ChatItem, MessageListOptions,
-        ProtectionStatus,
-    },
-    chatlist::Chatlist,
-    config::Config,
-    constants::DC_MSG_ID_DAYMARKER,
-    contact::{may_be_valid_addr, Contact, ContactId, Origin},
-    context::get_info,
-    ephemeral::Timer,
-    imex, location,
-    message::{self, delete_msgs, markseen_msgs, Message, MessageState, MsgId, Viewtype},
-    provider::get_provider_info,
-    qr,
-    qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg},
-    reaction::{get_msg_reactions, send_reaction},
-    securejoin,
-    stock_str::StockMessage,
-    webxdc::StatusUpdateSerial,
+use deltachat::chat::{
+    self, add_contact_to_chat, forward_msgs, get_chat_media, get_chat_msgs, get_chat_msgs_ex,
+    marknoticed_chat, remove_contact_from_chat, Chat, ChatId, ChatItem, MessageListOptions,
+    ProtectionStatus,
 };
+use deltachat::chatlist::Chatlist;
+use deltachat::config::Config;
+use deltachat::constants::DC_MSG_ID_DAYMARKER;
+use deltachat::contact::{may_be_valid_addr, Contact, ContactId, Origin};
+use deltachat::context::get_info;
+use deltachat::ephemeral::Timer;
+use deltachat::imex;
+use deltachat::location;
+use deltachat::message::get_msg_read_receipts;
+use deltachat::message::{
+    self, delete_msgs, markseen_msgs, Message, MessageState, MsgId, Viewtype,
+};
+use deltachat::provider::get_provider_info;
+use deltachat::qr::{self, Qr};
+use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
+use deltachat::reaction::{get_msg_reactions, send_reaction};
+use deltachat::securejoin;
+use deltachat::stock_str::StockMessage;
+use deltachat::webxdc::StatusUpdateSerial;
 use sanitize_filename::is_sanitized;
 use tokio::fs;
 use tokio::sync::{watch, Mutex, RwLock};
@@ -47,7 +47,7 @@ use types::provider_info::ProviderInfo;
 use types::reactions::JSONRPCReactions;
 use types::webxdc::WebxdcMessageInfo;
 
-use self::types::message::MessageLoadResult;
+use self::types::message::{MessageInfo, MessageLoadResult};
 use self::types::{
     chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
     location::JsonrpcLocation,
@@ -142,11 +142,7 @@ impl CommandApi {
     }
 }
 
-#[rpc(
-    all_positional,
-    ts_outdir = "typescript/generated",
-    openrpc_outdir = "openrpc"
-)]
+#[rpc(all_positional, ts_outdir = "typescript/generated")]
 impl CommandApi {
     /// Test function.
     async fn sleep(&self, delay: f64) {
@@ -157,12 +153,12 @@ impl CommandApi {
     //  Misc top level functions
     // ---------------------------------------------
 
-    /// Check if an email address is valid.
+    /// Checks if an email address is valid.
     async fn check_email_validity(&self, email: String) -> bool {
         may_be_valid_addr(&email)
     }
 
-    /// Get general system info.
+    /// Returns general system info.
     async fn get_system_info(&self) -> BTreeMap<&'static str, String> {
         get_info()
     }
@@ -223,13 +219,15 @@ impl CommandApi {
         Ok(accounts)
     }
 
+    /// Starts background tasks for all accounts.
     async fn start_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.start_io().await;
+        self.accounts.write().await.start_io().await;
         Ok(())
     }
 
+    /// Stops background tasks for all accounts.
     async fn stop_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.read().await.stop_io().await;
+        self.accounts.write().await.stop_io().await;
         Ok(())
     }
 
@@ -237,14 +235,16 @@ impl CommandApi {
     // Methods that work on individual accounts
     // ---------------------------------------------
 
-    async fn start_io(&self, id: u32) -> Result<()> {
-        let ctx = self.get_context(id).await?;
+    /// Starts background tasks for a single account.
+    async fn start_io(&self, account_id: u32) -> Result<()> {
+        let mut ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
 
-    async fn stop_io(&self, id: u32) -> Result<()> {
-        let ctx = self.get_context(id).await?;
+    /// Stops background tasks for a single account.
+    async fn stop_io(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         Ok(())
     }
@@ -311,11 +311,13 @@ impl CommandApi {
         ctx.get_info().await
     }
 
+    /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
         set_config(&ctx, &key, value.as_deref()).await
     }
 
+    /// Updates a batch of configuration values.
     async fn batch_set_config(
         &self,
         account_id: u32,
@@ -347,6 +349,7 @@ impl CommandApi {
         Ok(qr_object)
     }
 
+    /// Returns configuration value for the given key.
     async fn get_config(&self, account_id: u32, key: String) -> Result<Option<String>> {
         let ctx = self.get_context(account_id).await?;
         get_config(&ctx, &key).await
@@ -380,7 +383,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
+        let mut ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -675,7 +678,7 @@ impl CommandApi {
     ///     the Verified-Group-Invite protocol is offered in the QR code;
     ///     works for protected groups as well as for normal groups.
     ///     If not set, the Setup-Contact protocol is offered in the QR code.
-    ///     See https://countermitm.readthedocs.io/en/latest/new.html
+    ///     See https://securejoin.readthedocs.io/en/latest/new.html
     ///     for details about both protocols.
     ///
     /// return format: `[code, svg]`
@@ -704,7 +707,7 @@ impl CommandApi {
     ///
     /// Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
     ///
-    /// See https://countermitm.readthedocs.io/en/latest/new.html
+    /// See https://securejoin.readthedocs.io/en/latest/new.html
     /// for details about both protocols.
     ///
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
@@ -812,24 +815,12 @@ impl CommandApi {
     /// Create a new broadcast list.
     ///
     /// Broadcast lists are similar to groups on the sending device,
-    /// however, recipients get the messages in normal one-to-one chats
-    /// and will not be aware of other members.
+    /// however, recipients get the messages in a read-only chat
+    /// and will see who the other members are.
     ///
-    /// Replies to broadcasts go only to the sender
-    /// and not to all broadcast recipients.
-    /// Moreover, replies will not appear in the broadcast list
-    /// but in the one-to-one chat with the person answering.
-    ///
-    /// The name and the image of the broadcast list is set automatically
-    /// and is visible to the sender only.
-    /// Not asking for these data allows more focused creation
-    /// and we bypass the question who will get which data.
-    /// Also, many users will have at most one broadcast list
-    /// so, a generic name and image is sufficient at the first place.
-    ///
-    /// Later on, however, the name can be changed using dc_set_chat_name().
-    /// The image cannot be changed to have a unique, recognizable icon in the chat lists.
-    /// All in all, this is also what other messengers are doing here.
+    /// For historical reasons, this function does not take a name directly,
+    /// instead you have to set the name using dc_set_chat_name()
+    /// after creating the broadcast list.
     async fn create_broadcast_list(&self, account_id: u32) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
         chat::create_broadcast_list(&ctx)
@@ -1135,6 +1126,16 @@ impl CommandApi {
         MsgId::new(message_id).get_info(&ctx).await
     }
 
+    /// Returns additional information for single message.
+    async fn get_message_info_object(
+        &self,
+        account_id: u32,
+        message_id: u32,
+    ) -> Result<MessageInfo> {
+        let ctx = self.get_context(account_id).await?;
+        MessageInfo::from_msg_id(&ctx, MsgId::new(message_id)).await
+    }
+
     /// Returns contacts that sent read receipts and the time of reading.
     async fn get_message_read_receipts(
         &self,
@@ -1390,6 +1391,19 @@ impl CommandApi {
     //                   chat
     // ---------------------------------------------
 
+    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id` if it exists.
+    ///
+    /// If it does not exist, `None` is returned.
+    async fn get_chat_id_by_contact_id(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<Option<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let chat_id = ChatId::lookup_by_contact(&ctx, ContactId::new(contact_id)).await?;
+        Ok(chat_id.map(|id| id.to_u32()))
+    }
+
     /// Returns all message IDs of the given types in a chat.
     /// Typically used to show a gallery.
     ///
@@ -1753,6 +1767,9 @@ impl CommandApi {
         let mut msg = Message::new(Viewtype::Sticker);
         msg.set_file(&sticker_path, None);
 
+        // JSON-rpc does not need heuristics to turn [Viewtype::Sticker] into [Viewtype::Image]
+        msg.force_sticker();
+
         let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
         Ok(message_id.to_u32())
     }
@@ -1886,7 +1903,7 @@ impl CommandApi {
             .context("path conversion to string failed")
     }
 
-    /// save a sticker to a collection/folder in the account's sticker folder
+    /// Saves a sticker to a collection/folder in the account's sticker folder.
     async fn misc_save_sticker(
         &self,
         account_id: u32,
@@ -2039,13 +2056,19 @@ impl CommandApi {
         text: Option<String>,
         file: Option<String>,
         quoted_message_id: Option<u32>,
+        view_type: Option<MessageViewtype>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        let mut draft = Message::new(if file.is_some() {
-            Viewtype::File
-        } else {
-            Viewtype::Text
-        });
+        let mut draft = Message::new(view_type.map_or_else(
+            || {
+                if file.is_some() {
+                    Viewtype::File
+                } else {
+                    Viewtype::Text
+                }
+            },
+            |v| v.into(),
+        ));
         draft.set_text(text.unwrap_or_default());
         if let Some(file) = file {
             draft.set_file(file, None);
@@ -2065,6 +2088,23 @@ impl CommandApi {
 
         ChatId::new(chat_id).set_draft(&ctx, Some(&mut draft)).await
     }
+
+    // send the chat's current set draft
+    async fn misc_send_draft(&self, account_id: u32, chat_id: u32) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        if let Some(draft) = ChatId::new(chat_id).get_draft(&ctx).await? {
+            let mut draft = draft;
+            let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut draft)
+                .await?
+                .to_u32();
+            Ok(msg_id)
+        } else {
+            Err(anyhow!(
+                "chat with id {} doesn't have draft message",
+                chat_id
+            ))
+        }
+    }
 }
 
 // Helper functions (to prevent code duplication)

deltachat-jsonrpc/src/api/types/account.rs

@@ -7,7 +7,7 @@ use typescript_type_def::TypeDef;
 use super::color_int_to_hex_string;
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
-#[serde(tag = "type")]
+#[serde(tag = "kind")]
 pub enum Account {
     #[serde(rename_all = "camelCase")]
     Configured {

deltachat-jsonrpc/src/api/types/chat.rs

@@ -18,6 +18,17 @@ use super::contact::ContactObject;
 pub struct FullChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -31,6 +42,7 @@ pub struct FullChat {
     fresh_message_counter: usize,
     // is_group - please check over chat.type in frontend instead
     is_contact_request: bool,
+    is_protection_broken: bool,
     is_device_chat: bool,
     self_in_group: bool,
     is_muted: bool,
@@ -73,7 +85,7 @@ impl FullChat {
         let can_send = chat.can_send(context).await?;
 
         let was_seen_recently = if chat.get_type() == Chattype::Single {
-            match contact_ids.get(0) {
+            match contact_ids.first() {
                 Some(contact) => Contact::get_by_id(context, *contact)
                     .await
                     .context("failed to load contact for was_seen_recently")?
@@ -100,6 +112,7 @@ impl FullChat {
             color,
             fresh_message_counter,
             is_contact_request: chat.is_contact_request(),
+            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             self_in_group: contact_ids.contains(&ContactId::SELF),
             is_muted: chat.is_muted(),
@@ -126,6 +139,17 @@ impl FullChat {
 pub struct BasicChat {
     id: u32,
     name: String,
+
+    /// True if the chat is protected.
+    ///
+    /// UI should display a green checkmark
+    /// in the chat title,
+    /// in the chat profile title and
+    /// in the chatlist item
+    /// if chat protection is enabled.
+    /// UI should also display a green checkmark
+    /// in the contact profile
+    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -134,6 +158,7 @@ pub struct BasicChat {
     is_self_talk: bool,
     color: String,
     is_contact_request: bool,
+    is_protection_broken: bool,
     is_device_chat: bool,
     is_muted: bool,
 }
@@ -160,6 +185,7 @@ impl BasicChat {
             is_self_talk: chat.is_self_talk(),
             color,
             is_contact_request: chat.is_contact_request(),
+            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             is_muted: chat.is_muted(),
         })
@@ -167,10 +193,11 @@ impl BasicChat {
 }
 
 #[derive(Clone, Serialize, Deserialize, TypeDef, schemars::JsonSchema)]
+#[serde(tag = "kind")]
 pub enum MuteDuration {
     NotMuted,
     Forever,
-    Until(i64),
+    Until { duration: i64 },
 }
 
 impl MuteDuration {
@@ -178,13 +205,13 @@ impl MuteDuration {
         match self {
             MuteDuration::NotMuted => Ok(chat::MuteDuration::NotMuted),
             MuteDuration::Forever => Ok(chat::MuteDuration::Forever),
-            MuteDuration::Until(n) => {
-                if n <= 0 {
+            MuteDuration::Until { duration } => {
+                if duration <= 0 {
                     bail!("failed to read mute duration")
                 }
 
                 Ok(SystemTime::now()
-                    .checked_add(Duration::from_secs(n as u64))
+                    .checked_add(Duration::from_secs(duration as u64))
                     .map_or(chat::MuteDuration::Forever, chat::MuteDuration::Until))
             }
         }

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -15,7 +15,7 @@ use super::color_int_to_hex_string;
 use super::message::MessageViewtype;
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
-#[serde(tag = "type")]
+#[serde(tag = "kind")]
 pub enum ChatListItemFetchResult {
     #[serde(rename_all = "camelCase")]
     ChatListItem {
@@ -102,7 +102,7 @@ pub(crate) async fn get_chat_list_item_by_id(
     let self_in_group = chat_contacts.contains(&ContactId::SELF);
 
     let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
-        let contact = chat_contacts.get(0);
+        let contact = chat_contacts.first();
         let was_seen_recently = match contact {
             Some(contact) => Contact::get_by_id(ctx, *contact)
                 .await

deltachat-jsonrpc/src/api/types/contact.rs

@@ -1,5 +1,4 @@
 use anyhow::Result;
-use deltachat::contact::VerifiedStatus;
 use deltachat::context::Context;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
@@ -19,14 +18,36 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
+
+    /// True if the contact can be added to verified groups.
+    ///
+    /// If this is true
+    /// UI should display green checkmark after the contact name
+    /// in contact list items,
+    /// in chat member list items
+    /// and in profiles if no chat with the contact exist.
     is_verified: bool,
-    /// the address that verified this contact
-    verifier_addr: Option<String>,
-    /// the id of the contact that verified this contact
+
+    /// True if the contact profile title should have a green checkmark.
+    ///
+    /// This indicates whether 1:1 chat has a green checkmark
+    /// or will have a green checkmark if created.
+    is_profile_verified: bool,
+
+    /// The ID of the contact that verified this contact.
+    ///
+    /// If this is present,
+    /// display a green checkmark and "Introduced by ..."
+    /// string followed by the verifier contact name and address
+    /// in the contact profile.
     verifier_id: Option<u32>,
+
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
+
+    /// If the contact is a bot.
+    is_bot: bool,
 }
 
 impl ContactObject {
@@ -38,19 +59,13 @@ impl ContactObject {
             Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
             None => None,
         };
-        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+        let is_verified = contact.is_verified(context).await?;
+        let is_profile_verified = contact.is_profile_verified(context).await?;
 
-        let (verifier_addr, verifier_id) = if is_verified {
-            (
-                contact.get_verifier_addr(context).await?,
-                contact
-                    .get_verifier_id(context)
-                    .await?
-                    .map(|contact_id| contact_id.to_u32()),
-            )
-        } else {
-            (None, None)
-        };
+        let verifier_id = contact
+            .get_verifier_id(context)
+            .await?
+            .map(|contact_id| contact_id.to_u32());
 
         Ok(ContactObject {
             address: contact.get_addr().to_owned(),
@@ -64,10 +79,11 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
-            verifier_addr,
+            is_profile_verified,
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),
+            is_bot: contact.is_bot(),
         })
     }
 }

deltachat-jsonrpc/src/api/types/events.rs

@@ -22,61 +22,43 @@ impl From<CoreEvent> for Event {
 }
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
-#[serde(tag = "type")]
+#[serde(tag = "kind")]
 pub enum EventType {
     /// The library-user may write an informational string to the log.
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Info {
-        msg: String,
-    },
+    Info { msg: String },
 
     /// Emitted when SMTP connection is established and login was successful.
-    SmtpConnected {
-        msg: String,
-    },
+    SmtpConnected { msg: String },
 
     /// Emitted when IMAP connection is established and login was successful.
-    ImapConnected {
-        msg: String,
-    },
+    ImapConnected { msg: String },
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    SmtpMessageSent {
-        msg: String,
-    },
+    SmtpMessageSent { msg: String },
 
     /// Emitted when an IMAP message has been marked as deleted
-    ImapMessageDeleted {
-        msg: String,
-    },
+    ImapMessageDeleted { msg: String },
 
     /// Emitted when an IMAP message has been moved
-    ImapMessageMoved {
-        msg: String,
-    },
+    ImapMessageMoved { msg: String },
 
     /// Emitted before going into IDLE on the Inbox folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created
-    NewBlobFile {
-        file: String,
-    },
+    NewBlobFile { file: String },
 
     /// Emitted when an file in the $BLOBDIR was deleted
-    DeletedBlobFile {
-        file: String,
-    },
+    DeletedBlobFile { file: String },
 
     /// The library-user should write a warning string to the log.
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Warning {
-        msg: String,
-    },
+    Warning { msg: String },
 
     /// The library-user should report an error to the end-user.
     ///
@@ -88,18 +70,14 @@ pub enum EventType {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    Error {
-        msg: String,
-    },
+    Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// setChatName(), setChatProfileImage(),
     /// addContactToChat(), removeContactFromChat(),
     /// and messages sending functions.
-    ErrorSelfNotInGroup {
-        msg: String,
-    },
+    ErrorSelfNotInGroup { msg: String },
 
     /// Messages or chats changed.  One or more messages or chats changed for various
     /// reasons in the database:
@@ -110,10 +88,7 @@ pub enum EventType {
     /// `chatId` is set if only a single chat is affected by the changes, otherwise 0.
     /// `msgId` is set if only a single message is affected by the changes, otherwise 0.
     #[serde(rename_all = "camelCase")]
-    MsgsChanged {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgsChanged { chat_id: u32, msg_id: u32 },
 
     /// Reactions for the message changed.
     #[serde(rename_all = "camelCase")]
@@ -128,10 +103,7 @@ pub enum EventType {
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
     #[serde(rename_all = "camelCase")]
-    IncomingMsg {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    IncomingMsg { chat_id: u32, msg_id: u32 },
 
     /// Downloading a bunch of messages just finished. This is an experimental
     /// event to allow the UI to only show one notification per message bunch,
@@ -139,47 +111,31 @@ pub enum EventType {
     ///
     /// msg_ids contains the message ids.
     #[serde(rename_all = "camelCase")]
-    IncomingMsgBunch {
-        msg_ids: Vec<u32>,
-    },
+    IncomingMsgBunch { msg_ids: Vec<u32> },
 
     /// Messages were seen or noticed.
     /// chat id is always set.
     #[serde(rename_all = "camelCase")]
-    MsgsNoticed {
-        chat_id: u32,
-    },
+    MsgsNoticed { chat_id: u32 },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgDelivered {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDelivered { chat_id: u32, msg_id: u32 },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgFailed {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgFailed { chat_id: u32, msg_id: u32 },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgRead {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgRead { chat_id: u32, msg_id: u32 },
 
     /// A single message is deleted.
     #[serde(rename_all = "camelCase")]
-    MsgDeleted {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDeleted { chat_id: u32, msg_id: u32 },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -189,24 +145,17 @@ pub enum EventType {
     /// This event does not include ephemeral timer modification, which
     /// is a separate event.
     #[serde(rename_all = "camelCase")]
-    ChatModified {
-        chat_id: u32,
-    },
+    ChatModified { chat_id: u32 },
 
     /// Chat ephemeral timer changed.
     #[serde(rename_all = "camelCase")]
-    ChatEphemeralTimerModified {
-        chat_id: u32,
-        timer: u32,
-    },
+    ChatEphemeralTimerModified { chat_id: u32, timer: u32 },
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[serde(rename_all = "camelCase")]
-    ContactsChanged {
-        contact_id: Option<u32>,
-    },
+    ContactsChanged { contact_id: Option<u32> },
 
     /// Location of one or more contact has changed.
     ///
@@ -214,9 +163,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     this parameter is set to `None`.
     #[serde(rename_all = "camelCase")]
-    LocationChanged {
-        contact_id: Option<u32>,
-    },
+    LocationChanged { contact_id: Option<u32> },
 
     /// Inform about the configuration progress started by configure().
     ConfigureProgress {
@@ -234,9 +181,7 @@ pub enum EventType {
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexProgress {
-        progress: usize,
-    },
+    ImexProgress { progress: usize },
 
     /// A file has been exported. A file has been written by imex().
     /// This event may be sent multiple times by a single call to imex().
@@ -246,9 +191,7 @@ pub enum EventType {
     ///
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexFileWritten {
-        path: String,
-    },
+    ImexFileWritten { path: String },
 
     /// Progress information of a secure-join handshake from the view of the inviter
     /// (Alice, the person who shows the QR code).
@@ -263,10 +206,7 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[serde(rename_all = "camelCase")]
-    SecurejoinInviterProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -277,10 +217,7 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[serde(rename_all = "camelCase")]
-    SecurejoinJoinerProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -288,8 +225,17 @@ pub enum EventType {
     /// getConnectivityHtml() for details.
     ConnectivityChanged,
 
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: String,
+    },
+
     #[serde(rename_all = "camelCase")]
     WebxdcStatusUpdate {
         msg_id: u32,
@@ -298,9 +244,7 @@ pub enum EventType {
 
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
-    WebxdcInstanceDeleted {
-        msg_id: u32,
-    },
+    WebxdcInstanceDeleted { msg_id: u32 },
 }
 
 impl From<CoreEventType> for EventType {
@@ -396,6 +340,9 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ConnectivityChanged => ConnectivityChanged,
             CoreEventType::SelfavatarChanged => SelfavatarChanged,
+            CoreEventType::ConfigSynced { key } => ConfigSynced {
+                key: key.to_string(),
+            },
             CoreEventType::WebxdcStatusUpdate {
                 msg_id,
                 status_update_serial,

deltachat-jsonrpc/src/api/types/message.rs

@@ -19,7 +19,7 @@ use super::reactions::JSONRPCReactions;
 use super::webxdc::WebxdcMessageInfo;
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
-#[serde(rename_all = "camelCase", tag = "variant")]
+#[serde(rename_all = "camelCase", tag = "kind")]
 pub enum MessageLoadResult {
     Message(MessageObject),
     LoadingError { error: String },
@@ -345,6 +345,7 @@ pub enum SystemMessageType {
     SecurejoinMessage,
     LocationStreamingEnabled,
     LocationOnly,
+    InvalidUnencryptedMail,
 
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged,
@@ -385,6 +386,7 @@ impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
             SystemMessage::MultiDeviceSync => SystemMessageType::MultiDeviceSync,
             SystemMessage::WebxdcStatusUpdate => SystemMessageType::WebxdcStatusUpdate,
             SystemMessage::WebxdcInfoMessage => SystemMessageType::WebxdcInfoMessage,
+            SystemMessage::InvalidUnencryptedMail => SystemMessageType::InvalidUnencryptedMail,
         }
     }
 }
@@ -552,3 +554,71 @@ pub struct MessageReadReceipt {
     pub contact_id: u32,
     pub timestamp: i64,
 }
+
+#[derive(Serialize, TypeDef, schemars::JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct MessageInfo {
+    rawtext: String,
+    ephemeral_timer: EphemeralTimer,
+    /// When message is ephemeral this contains the timestamp of the message expiry
+    ephemeral_timestamp: Option<i64>,
+    error: Option<String>,
+    rfc724_mid: String,
+    server_urls: Vec<String>,
+    hop_info: Option<String>,
+}
+
+impl MessageInfo {
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+        let message = Message::load_from_db(context, msg_id).await?;
+        let rawtext = msg_id.rawtext(context).await?;
+        let ephemeral_timer = message.get_ephemeral_timer().into();
+        let ephemeral_timestamp = match message.get_ephemeral_timer() {
+            deltachat::ephemeral::Timer::Disabled => None,
+            deltachat::ephemeral::Timer::Enabled { .. } => Some(message.get_ephemeral_timestamp()),
+        };
+
+        let server_urls =
+            MsgId::get_info_server_urls(context, message.rfc724_mid().to_owned()).await?;
+
+        let hop_info = msg_id.hop_info(context).await?;
+
+        Ok(Self {
+            rawtext,
+            ephemeral_timer,
+            ephemeral_timestamp,
+            error: message.error(),
+            rfc724_mid: message.rfc724_mid().to_owned(),
+            server_urls,
+            hop_info,
+        })
+    }
+}
+
+#[derive(
+    Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize, TypeDef, schemars::JsonSchema,
+)]
+#[serde(rename_all = "camelCase", tag = "variant")]
+pub enum EphemeralTimer {
+    /// Timer is disabled.
+    Disabled,
+
+    /// Timer is enabled.
+    Enabled {
+        /// Timer duration in seconds.
+        ///
+        /// The value cannot be 0.
+        duration: u32,
+    },
+}
+
+impl From<deltachat::ephemeral::Timer> for EphemeralTimer {
+    fn from(value: deltachat::ephemeral::Timer) -> Self {
+        match value {
+            deltachat::ephemeral::Timer::Disabled => EphemeralTimer::Disabled,
+            deltachat::ephemeral::Timer::Enabled { duration } => {
+                EphemeralTimer::Enabled { duration }
+            }
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -6,6 +6,8 @@ use typescript_type_def::TypeDef;
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ProviderInfo {
+    /// Unique ID, corresponding to provider database filename.
+    pub id: String,
     pub before_login_hint: String,
     pub overview_page: String,
     pub status: u32, // in reality this is an enum, but for simplicity and because it gets converted into a number anyway, we use an u32 here.
@@ -14,6 +16,7 @@ pub struct ProviderInfo {
 impl ProviderInfo {
     pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
         provider.map(|p| ProviderInfo {
+            id: p.id.to_owned(),
             before_login_hint: p.before_login_hint.to_owned(),
             overview_page: p.overview_page.to_owned(),
             status: p.status.to_u32().unwrap(),

deltachat-jsonrpc/src/api/types/qr.rs

@@ -4,7 +4,7 @@ use typescript_type_def::TypeDef;
 
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename = "Qr", rename_all = "camelCase")]
-#[serde(tag = "type")]
+#[serde(tag = "kind")]
 pub enum QrObject {
     AskVerifyContact {
         contact_id: u32,

deltachat-jsonrpc/src/lib.rs

@@ -13,10 +13,11 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
         let tmp_dir = TempDir::new().unwrap().path().into();
-        let accounts = Accounts::new(tmp_dir).await?;
+        let writable = true;
+        let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, mut receiver) = unbounded::<String>();
+        let (sender, receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -35,17 +36,17 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
+            let result = receiver.recv().await?;
             println!("{result:?}");
-            assert_eq!(result, Some(response.to_owned()));
+            assert_eq!(result, response.to_owned());
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
+            let result = receiver.recv().await?;
             println!("{result:?}");
-            assert_eq!(result, Some(response.to_owned()));
+            assert_eq!(result, response.to_owned());
         }
 
         Ok(())
@@ -54,10 +55,11 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn test_batch_set_config() -> anyhow::Result<()> {
         let tmp_dir = TempDir::new().unwrap().path().into();
-        let accounts = Accounts::new(tmp_dir).await?;
+        let writable = true;
+        let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, mut receiver) = unbounded::<String>();
+        let (sender, receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -76,15 +78,15 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
-            assert_eq!(result, Some(response.to_owned()));
+            let result = receiver.recv().await?;
+            assert_eq!(result, response.to_owned());
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
             session.handle_incoming(request).await;
-            let result = receiver.next().await;
-            assert_eq!(result, Some(response.to_owned()));
+            let result = receiver.recv().await?;
+            assert_eq!(result, response.to_owned());
         }
 
         Ok(())

deltachat-jsonrpc/src/webserver.rs

@@ -19,7 +19,8 @@ async fn main() -> Result<(), std::io::Error> {
         .map(|port| port.parse::<u16>().expect("DC_PORT must be a number"))
         .unwrap_or(DEFAULT_PORT);
     log::info!("Starting with accounts directory `{path}`.");
-    let accounts = Accounts::new(PathBuf::from(&path)).await.unwrap();
+    let writable = true;
+    let accounts = Accounts::new(PathBuf::from(&path), writable).await.unwrap();
     let state = CommandApi::new(accounts);
 
     let app = Router::new()
@@ -27,15 +28,13 @@ async fn main() -> Result<(), std::io::Error> {
         .layer(Extension(state.clone()));
 
     tokio::spawn(async move {
-        state.accounts.read().await.start_io().await;
+        state.accounts.write().await.start_io().await;
     });
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));
     log::info!("JSON-RPC WebSocket server listening on {}", addr);
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await
-        .unwrap();
+    let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+    axum::serve(listener, app).await.unwrap();
 
     Ok(())
 }

deltachat-jsonrpc/typescript/example/example.ts

@@ -35,7 +35,7 @@ async function run() {
     const accounts = await client.rpc.getAllAccounts();
     console.log("accounts loaded", accounts);
     for (const account of accounts) {
-      if (account.type === "Configured") {
+      if (account.kind === "Configured") {
         write(
           $head,
           `<a href="#" onclick="selectDeltaAccount(${account.id})">
@@ -57,7 +57,7 @@ async function run() {
     clear($main);
     const selectedAccount = SELECTED_ACCOUNT;
     const info = await client.rpc.getAccountInfo(selectedAccount);
-    if (info.type !== "Configured") {
+    if (info.kind !== "Configured") {
       return write($main, "Account is not configured");
     }
     write($main, `<h2>${info.addr!}</h2>`);
@@ -81,8 +81,7 @@ async function run() {
         messageIds
       );
       for (const [_messageId, message] of Object.entries(messages)) {
-        if (message.variant === "message")
-          write($main, `<p>${message.text}</p>`);
+        if (message.kind === "message") write($main, `<p>${message.text}</p>`);
         else write($main, `<p>loading error: ${message.error}</p>`);
       }
     }
@@ -93,9 +92,9 @@ async function run() {
       $side,
       `
         <p class="message">
-          [<strong>${event.type}</strong> on account ${accountId}]<br>
+          [<strong>${event.kind}</strong> on account ${accountId}]<br>
           <em>f1:</em> ${JSON.stringify(
-            Object.assign({}, event, { type: undefined })
+            Object.assign({}, event, { kind: undefined })
           )}
         </p>`
     );

deltachat-jsonrpc/typescript/package.json

@@ -9,7 +9,6 @@
     "@types/chai": "^4.2.21",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^9.0.0",
-    "@types/node-fetch": "^2.5.7",
     "@types/ws": "^7.2.4",
     "c8": "^7.10.0",
     "chai": "^4.3.4",
@@ -17,7 +16,6 @@
     "esbuild": "^0.17.9",
     "http-server": "^14.1.1",
     "mocha": "^9.1.1",
-    "node-fetch": "^2.6.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.6.2",
     "typedoc": "^0.23.2",
@@ -55,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.124.1"
+  "version": "1.133.2"
 }

deltachat-jsonrpc/typescript/src/client.ts

@@ -6,22 +6,22 @@ import { WebsocketTransport, BaseTransport, Request } from "yerpc";
 import { TinyEmitter } from "@deltachat/tiny-emitter";
 
 type Events = { ALL: (accountId: number, event: EventType) => void } & {
-  [Property in EventType["type"]]: (
+  [Property in EventType["kind"]]: (
     accountId: number,
-    event: Extract<EventType, { type: Property }>
+    event: Extract<EventType, { kind: Property }>
   ) => void;
 };
 
 type ContextEvents = { ALL: (event: EventType) => void } & {
-  [Property in EventType["type"]]: (
-    event: Extract<EventType, { type: Property }>
+  [Property in EventType["kind"]]: (
+    event: Extract<EventType, { kind: Property }>
   ) => void;
 };
 
 export type DcEvent = EventType;
-export type DcEventType<T extends EventType["type"]> = Extract<
+export type DcEventType<T extends EventType["kind"]> = Extract<
   EventType,
-  { type: T }
+  { kind: T }
 >;
 
 export class BaseDeltaChat<
@@ -46,12 +46,12 @@ export class BaseDeltaChat<
     while (true) {
       const event = await this.rpc.getNextEvent();
       //@ts-ignore
-      this.emit(event.event.type, event.contextId, event.event);
+      this.emit(event.event.kind, event.contextId, event.event);
       this.emit("ALL", event.contextId, event.event);
 
       if (this.contextEmitters[event.contextId]) {
         this.contextEmitters[event.contextId].emit(
-          event.event.type,
+          event.event.kind,
           //@ts-ignore
           event.event as any
         );

deltachat-jsonrpc/typescript/test/basic.ts

@@ -79,6 +79,9 @@ describe("basic tests", () => {
       accountId = await dc.rpc.addAccount();
     });
     it("should block and unblock contact", async function () {
+      // Cannot send sync messages to self as we do not have a self address.
+      await dc.rpc.setConfig(accountId, "sync_msgs", "0");
+
       const contactId = await dc.rpc.createContact(
         accountId,
         "example@delta.chat",

deltachat-jsonrpc/typescript/test/online.ts

@@ -13,27 +13,27 @@ describe("online tests", function () {
 
   before(async function () {
     this.timeout(60000);
-    if (!process.env.DCC_NEW_TMP_EMAIL) {
+    if (!process.env.CHATMAIL_DOMAIN) {
       if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
         console.error(
-          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
+          "CAN NOT RUN COVERAGE correctly: Missing CHATMAIL_DOMAIN environment variable!\n\n",
           "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
         );
         process.exit(1);
       }
       console.log(
-        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests"
+        "Missing CHATMAIL_DOMAIN environment variable!, skip integration tests"
       );
       this.skip();
     }
     serverHandle = await startServer();
     dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout, true);
 
-    dc.on("ALL", (contextId, { type }) => {
-      if (type !== "Info") console.log(contextId, type);
+    dc.on("ALL", (contextId, { kind }) => {
+      if (kind !== "Info") console.log(contextId, kind);
     });
 
-    account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    account1 = createTempUser(process.env.CHATMAIL_DOMAIN);
     if (!account1 || !account1.email || !account1.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"
@@ -41,7 +41,7 @@ describe("online tests", function () {
       this.skip();
     }
 
-    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    account2 = createTempUser(process.env.CHATMAIL_DOMAIN);
     if (!account2 || !account2.email || !account2.password) {
       console.log(
         "We didn't got back an account2 from the api, skip integration tests"
@@ -148,7 +148,7 @@ describe("online tests", function () {
       waitForEvent(dc, "IncomingMsg", accountId1),
     ]);
     dc.rpc.miscSendTextMessage(accountId2, chatId, "super secret message");
-    // Check if answer arives at A and if it is encrypted
+    // Check if answer arrives at A and if it is encrypted
     await eventPromise2;
 
     const messageId = (
@@ -177,12 +177,12 @@ describe("online tests", function () {
   });
 });
 
-async function waitForEvent<T extends DcEvent["type"]>(
+async function waitForEvent<T extends DcEvent["kind"]>(
   dc: DeltaChat,
   eventType: T,
   accountId: number,
   timeout: number = EVENT_TIMEOUT
-): Promise<Extract<DcEvent, { type: T }>> {
+): Promise<Extract<DcEvent, { kind: T }>> {
   return new Promise((resolve, reject) => {
     const rejectTimeout = setTimeout(
       () => reject(new Error("Timeout reached before event came in")),

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -2,7 +2,6 @@ import { tmpdir } from "os";
 import { join, resolve } from "path";
 import { mkdtemp, rm } from "fs/promises";
 import { spawn, exec } from "child_process";
-import fetch from "node-fetch";
 import { Readable, Writable } from "node:stream";
 
 export type RpcServerHandle = {
@@ -57,15 +56,14 @@ export async function startServer(): Promise<RpcServerHandle> {
   };
 }
 
-export async function createTempUser(url: string) {
-  const response = await fetch(url, {
-    method: "POST",
-    headers: {
-      "cache-control": "no-cache",
-    },
-  });
-  if (!response.ok) throw new Error("Received invalid response");
-  return response.json();
+export function createTempUser(chatmailDomain: String) {
+  const charset = "2345789acdefghjkmnpqrstuvwxyz";
+  let user = "ci-";
+  for (let i = 0; i < 6; i++) {
+    user += charset[Math.floor(Math.random() * charset.length)];
+  }
+  const email = user + "@" + chatmailDomain;
+  return { email: email, password: user + "$" + user };
 }
 
 function getTargetDir(): Promise<string> {

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.124.1"
+version = "1.133.2"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -9,10 +9,10 @@ ansi_term = "0.12.1"
 anyhow = "1"
 deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
-log = "0.4.19"
+log = "0.4.20"
 pretty_env_logger = "0.5"
-rusqlite = "0.29"
-rustyline = "12"
+rusqlite = "0.30"
+rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 
 [features]

deltachat-repl/src/cmdline.rs

@@ -18,6 +18,7 @@ use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
 use deltachat::message::{self, Message, MessageState, MsgId, Viewtype};
+use deltachat::mimeparser::SystemMessage;
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::reaction::send_reaction;
@@ -210,7 +211,17 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         } else {
             "[FRESH]"
         },
-        if msg.is_info() { "[INFO]" } else { "" },
+        if msg.is_info() {
+            if msg.get_info_type() == SystemMessage::ChatProtectionEnabled {
+                "[INFO 🛡️]"
+            } else if msg.get_info_type() == SystemMessage::ChatProtectionDisabled {
+                "[INFO 🛡️❌]"
+            } else {
+                "[INFO]"
+            }
+        } else {
+            ""
+        },
         if msg.get_viewtype() == Viewtype::VideochatInvitation {
             format!(
                 "[VIDEOCHAT-INVITATION: {}, type={}]",
@@ -273,13 +284,8 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
         let addr = contact.get_addr();
-        let verified_state = contact.is_verified(context).await?;
-        let verified_str = if VerifiedStatus::Unverified != verified_state {
-            if verified_state == VerifiedStatus::BidirectVerified {
-                " √√"
-            } else {
-                " √"
-            }
+        let verified_str = if contact.is_verified(context).await? {
+            " √"
         } else {
             ""
         };
@@ -395,8 +401,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unpin <chat-id>\n\
                  mute <chat-id> [<seconds>]\n\
                  unmute <chat-id>\n\
-                 protect <chat-id>\n\
-                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
                  accept <chat-id>\n\
                  decline <chat-id>\n\
@@ -1071,20 +1075,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             };
             chat::set_muted(&context, chat_id, duration).await?;
         }
-        "protect" | "unprotect" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id
-                .set_protection(
-                    &context,
-                    match arg0 {
-                        "protect" => ProtectionStatus::Protected,
-                        "unprotect" => ProtectionStatus::Unprotected,
-                        _ => unreachable!("arg0={:?}", arg0),
-                    },
-                )
-                .await?;
-        }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);

deltachat-repl/src/main.rs

@@ -299,8 +299,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize) -> bool {
-        self.highlighter.highlight_char(line, pos)
+    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
+        self.highlighter.highlight_char(line, pos, forced)
     }
 }
 
@@ -401,7 +401,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    ctx: Context,
+    mut ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
@@ -481,7 +481,10 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    let _ = pretty_env_logger::try_init();
+    pretty_env_logger::formatted_timed_builder()
+        .parse_default_env()
+        .filter_module("deltachat_repl", log::LevelFilter::Info)
+        .init();
 
     let args = std::env::args().collect();
     start(args).await?;

deltachat-rpc-client/README.md

@@ -25,7 +25,7 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `PATH="../target/debug:$PATH" tox`.
+2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 

deltachat-rpc-client/examples/echobot_advanced.py

@@ -14,9 +14,9 @@ hooks = events.HookCollection()
 
 @hooks.on(events.RawEvent)
 def log_event(event):
-    if event.type == EventType.INFO:
+    if event.kind == EventType.INFO:
         logging.info(event.msg)
-    elif event.type == EventType.WARNING:
+    elif event.kind == EventType.WARNING:
         logging.warning(event.msg)
 
 

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -40,13 +40,13 @@ def main():
 
         while True:
             event = account.wait_for_event()
-            if event["type"] == EventType.INFO:
+            if event["kind"] == EventType.INFO:
                 logging.info("%s", event["msg"])
-            elif event["type"] == EventType.WARNING:
+            elif event["kind"] == EventType.WARNING:
                 logging.warning("%s", event["msg"])
-            elif event["type"] == EventType.ERROR:
+            elif event["kind"] == EventType.ERROR:
                 logging.error("%s", event["msg"])
-            elif event["type"] == EventType.INCOMING_MSG:
+            elif event["kind"] == EventType.INCOMING_MSG:
                 logging.info("Got an incoming message")
                 process_messages()
 

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -4,7 +4,7 @@ from warnings import warn
 
 from ._utils import AttrDict
 from .chat import Chat
-from .const import ChatlistFlag, ContactFlag, SpecialContactId
+from .const import ChatlistFlag, ContactFlag, EventType, SpecialContactId
 from .contact import Contact
 from .message import Message
 
@@ -111,6 +111,20 @@ class Account:
         contacts = self._rpc.get_blocked_contacts(self.id)
         return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
 
+    def get_chat_by_contact(self, contact: Union[int, Contact]) -> Optional[Chat]:
+        """Return 1:1 chat for a contact if it exists."""
+        if isinstance(contact, Contact):
+            assert contact.account == self
+            contact_id = contact.id
+        elif isinstance(contact, int):
+            contact_id = contact
+        else:
+            raise ValueError(f"{contact!r} is not a contact")
+        chat_id = self._rpc.get_chat_id_by_contact_id(self.id, contact_id)
+        if chat_id:
+            return Chat(self, chat_id)
+        return None
+
     def get_contacts(
         self,
         query: Optional[str] = None,
@@ -204,7 +218,7 @@ class Account:
         The function returns immediately and the handshake runs in background, sending
         and receiving several messages.
         Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
-        See https://countermitm.readthedocs.io/en/latest/new.html for protocol details.
+        See https://securejoin.readthedocs.io/en/latest/new.html for protocol details.
 
         :param qrdata: The text of the scanned QR code.
         """
@@ -250,6 +264,25 @@ class Account:
         next_msg_ids = self._rpc.wait_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
+    def wait_for_incoming_msg_event(self):
+        """Wait for incoming message event and return it."""
+        while True:
+            event = self.wait_for_event()
+            if event.kind == EventType.INCOMING_MSG:
+                return event
+
+    def wait_for_securejoin_inviter_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
+                break
+
+    def wait_for_securejoin_joiner_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+                break
+
     def get_fresh_messages_in_arrival_order(self) -> List[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(
@@ -267,3 +300,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -54,14 +54,14 @@ class Chat:
         """
         if duration is not None:
             assert duration > 0, "Invalid duration"
-            dur: Union[str, dict] = {"Until": duration}
+            dur: dict = {"kind": "Until", "duration": duration}
         else:
-            dur = "Forever"
+            dur = {"kind": "Forever"}
         self._rpc.set_chat_mute_duration(self.account.id, self.id, dur)
 
     def unmute(self) -> None:
         """Unmute this chat."""
-        self._rpc.set_chat_mute_duration(self.account.id, self.id, "NotMuted")
+        self._rpc.set_chat_mute_duration(self.account.id, self.id, {"kind": "NotMuted"})
 
     def pin(self) -> None:
         """Pin this chat."""
@@ -160,11 +160,12 @@ class Chat:
         text: Optional[str] = None,
         file: Optional[str] = None,
         quoted_msg: Optional[int] = None,
+        viewtype: Optional[str] = None,
     ) -> None:
         """Set draft message."""
         if isinstance(quoted_msg, Message):
             quoted_msg = quoted_msg.id
-        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg)
+        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg, viewtype)
 
     def remove_draft(self) -> None:
         """Remove draft message."""

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -3,7 +3,6 @@ import logging
 from typing import (
     TYPE_CHECKING,
     Callable,
-    Coroutine,
     Dict,
     Iterable,
     Optional,
@@ -92,7 +91,7 @@ class Client:
         """Process events forever."""
         self.run_until(lambda _: False)
 
-    def run_until(self, func: Callable[[AttrDict], Union[bool, Coroutine]]) -> AttrDict:
+    def run_until(self, func: Callable[[AttrDict], bool]) -> AttrDict:
         """Process events until the given callable evaluates to True.
 
         The callable should accept an AttrDict object representing the
@@ -105,10 +104,10 @@ class Client:
         self._process_messages()  # Process old messages.
         while True:
             event = self.account.wait_for_event()
-            event["type"] = EventType(event.type)
+            event["kind"] = EventType(event.kind)
             event["account"] = self.account
             self._on_event(event)
-            if event.type == EventType.INCOMING_MSG:
+            if event.kind == EventType.INCOMING_MSG:
                 self._process_messages()
 
             stop = func(event)
@@ -196,7 +195,7 @@ class Client:
 
 
 class Bot(Client):
-    """Simple bot implementation that listent to events of a single account."""
+    """Simple bot implementation that listens to events of a single account."""
 
     def configure(self, email: str, password: str, **kwargs) -> None:
         kwargs.setdefault("bot", "1")

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -79,7 +79,7 @@ class RawEvent(EventFilter):
         return False
 
     def filter(self, event: "AttrDict") -> bool:
-        if self.types and event.type not in self.types:
+        if self.types and event.kind not in self.types:
             return False
         return self._call_func(event)
 

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -42,6 +42,10 @@ class Message:
             return AttrDict(reactions)
         return None
 
+    def get_sender_contact(self) -> Contact:
+        from_id = self.get_snapshot().from_id
+        return self.account.get_contact_by_id(from_id)
+
     def mark_seen(self) -> None:
         """Mark the message as seen."""
         self._rpc.markseen_msgs(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,6 +1,5 @@
-import json
 import os
-import urllib.request
+import random
 from typing import AsyncGenerator, List, Optional
 
 import pytest
@@ -10,12 +9,11 @@ from .rpc import Rpc
 
 
 def get_temp_credentials() -> dict:
-    url = os.getenv("DCC_NEW_TMP_EMAIL")
-    assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
-
-    request = urllib.request.Request(url, method="POST")
-    with urllib.request.urlopen(request, timeout=60) as f:
-        return json.load(f)
+    domain = os.getenv("CHATMAIL_DOMAIN")
+    username = "ci-" + "".join(random.choice("2345789acdefghjkmnpqrstuvwxyz") for i in range(6))
+    password = f"{username}${username}"
+    addr = f"{username}@{domain}"
+    return {"email": addr, "password": password}
 
 
 class ACFactory:
@@ -23,7 +21,9 @@ class ACFactory:
         self.deltachat = deltachat
 
     def get_unconfigured_account(self) -> Account:
-        return self.deltachat.add_account()
+        account = self.deltachat.add_account()
+        account.set_config("verified_one_on_one_chats", "1")
+        return account
 
     def get_unconfigured_bot(self) -> Bot:
         return Bot(self.get_unconfigured_account())
@@ -54,13 +54,23 @@ class ACFactory:
         account.start_io()
         while True:
             event = account.wait_for_event()
-            if event.type == EventType.IMAP_INBOX_IDLE:
+            if event.kind == EventType.IMAP_INBOX_IDLE:
                 break
         return account
 
     def get_online_accounts(self, num: int) -> List[Account]:
         return [self.get_online_account() for _ in range(num)]
 
+    def resetup_account(self, ac: Account) -> Account:
+        """Resetup account from scratch, losing the encryption key."""
+        ac.stop_io()
+        ac_clone = self.get_unconfigured_account()
+        for i in ["addr", "mail_pw"]:
+            ac_clone.set_config(i, ac.get_config(i))
+        ac.remove()
+        ac_clone.configure()
+        return ac_clone
+
     def send_message(
         self,
         to_account: Account,
@@ -95,7 +105,7 @@ class ACFactory:
             group=group,
         )
 
-        return to_client.run_until(lambda e: e.type == EventType.INCOMING_MSG)
+        return to_client.run_until(lambda e: e.kind == EventType.INCOMING_MSG)
 
 
 @pytest.fixture()

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,3 +1,4 @@
+import itertools
 import json
 import logging
 import os
@@ -5,7 +6,7 @@ import subprocess
 import sys
 from queue import Queue
 from threading import Event, Thread
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Iterator, Optional
 
 
 class JsonRpcError(Exception):
@@ -23,7 +24,7 @@ class Rpc:
 
         self._kwargs = kwargs
         self.process: subprocess.Popen
-        self.id: int
+        self.id_iterator: Iterator[int]
         self.event_queues: Dict[int, Queue]
         # Map from request ID to `threading.Event`.
         self.request_events: Dict[int, Event]
@@ -54,7 +55,7 @@ class Rpc:
                 preexec_fn=os.setpgrp,  # noqa: PLW1509
                 **self._kwargs,
             )
-        self.id = 0
+        self.id_iterator = itertools.count(start=1)
         self.event_queues = {}
         self.request_events = {}
         self.request_results = {}
@@ -131,7 +132,9 @@ class Rpc:
                 event = self.get_next_event()
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
-                queue.put(event["event"])
+                event = event["event"]
+                logging.debug("account_id=%d got an event %s", account_id, event)
+                queue.put(event)
         except Exception:
             # Log an exception if the event loop dies.
             logging.exception("Exception in the event loop")
@@ -143,14 +146,12 @@ class Rpc:
 
     def __getattr__(self, attr: str):
         def method(*args) -> Any:
-            self.id += 1
-            request_id = self.id
-
+            request_id = next(self.id_iterator)
             request = {
                 "jsonrpc": "2.0",
                 "method": attr,
                 "params": args,
-                "id": self.id,
+                "id": request_id,
             }
             event = Event()
             self.request_events[request_id] = event

deltachat-rpc-client/tests/test_securejoin.py

@@ -0,0 +1,566 @@
+import logging
+
+import pytest
+from deltachat_rpc_client import Chat, SpecialContactId
+
+
+def test_qr_setup_contact(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+
+    alice.wait_for_securejoin_inviter_success()
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    bob.wait_for_securejoin_joiner_success()
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+    # Test that if Bob changes the key, backwards verification is lost.
+    logging.info("Bob 2 is created")
+    bob2 = acfactory.new_configured_account()
+    bob2.export_self_keys(tmp_path)
+
+    logging.info("Bob imports a key")
+    bob.import_self_keys(tmp_path / "private-key-default.asc")
+
+    assert bob.get_config("key_id") == "2"
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert not bob_contact_alice_snapshot.is_verified
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect):
+    alice, bob = acfactory.get_online_accounts(2)
+
+    logging.info("Alice creates a verified group")
+    alice_chat = alice.create_group("Verified group", protect=protect)
+    assert alice_chat.get_basic_snapshot().is_protected == protect
+
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+
+    # Check that at least some of the handshake messages are deleted.
+    for ac in [alice, bob]:
+        while True:
+            event = ac.wait_for_event()
+            if event["kind"] == "ImapMessageDeleted":
+                break
+
+    alice.wait_for_securejoin_inviter_success()
+
+    # Test that Alice verified Bob's profile.
+    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
+    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
+    assert alice_contact_bob_snapshot.is_verified
+
+    bob.wait_for_securejoin_joiner_success()
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
+    assert snapshot.chat.get_basic_snapshot().is_protected == protect
+
+    # Test that Bob verified Alice's profile.
+    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert bob_contact_alice_snapshot.is_verified
+
+
+def test_qr_securejoin_contact_request(acfactory) -> None:
+    """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+    bob_chat_alice = snapshot.chat
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+    alice_chat = alice.create_group("Verified group", protect=True)
+    logging.info("Bob joins verified group")
+    qr_code, _svg = alice_chat.get_qr_code()
+    bob.secure_join(qr_code)
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+            break
+
+    # Chat stays being a contact request.
+    assert bob_chat_alice.get_basic_snapshot().is_contact_request
+
+
+def test_qr_readreceipt(acfactory) -> None:
+    alice, bob, charlie = acfactory.get_online_accounts(3)
+
+    logging.info("Bob and Charlie setup contact with Alice")
+    qr_code, _svg = alice.get_qr_code()
+
+    bob.secure_join(qr_code)
+    charlie.secure_join(qr_code)
+
+    for joiner in [bob, charlie]:
+        joiner.wait_for_securejoin_joiner_success()
+
+    logging.info("Alice creates a verified group")
+    group = alice.create_group("Group", protect=True)
+
+    bob_addr = bob.get_config("addr")
+    charlie_addr = charlie.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_contact_charlie = alice.create_contact(charlie_addr, "Charlie")
+
+    group.add_contact(alice_contact_bob)
+    group.add_contact(alice_contact_charlie)
+
+    # Promote a group.
+    group.send_message(text="Hello")
+
+    logging.info("Bob and Charlie receive a group")
+
+    bob_msg_id = bob.wait_for_incoming_msg_event().msg_id
+    bob_message = bob.get_message_by_id(bob_msg_id)
+    bob_snapshot = bob_message.get_snapshot()
+    assert bob_snapshot.text == "Hello"
+
+    # Charlie receives the same "Hello" message as Bob.
+    charlie.wait_for_incoming_msg_event()
+
+    logging.info("Bob sends a message to the group")
+
+    bob_out_message = bob_snapshot.chat.send_message(text="Hi from Bob!")
+
+    charlie_msg_id = charlie.wait_for_incoming_msg_event().msg_id
+    charlie_message = charlie.get_message_by_id(charlie_msg_id)
+    charlie_snapshot = charlie_message.get_snapshot()
+    assert charlie_snapshot.text == "Hi from Bob!"
+
+    bob_contact_charlie = bob.create_contact(charlie_addr, "Charlie")
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+    logging.info("Charlie reads Bob's message")
+    charlie_message.mark_seen()
+
+    while True:
+        event = bob.wait_for_event()
+        if event["kind"] == "MsgRead" and event["msg_id"] == bob_out_message.id:
+            break
+
+    # Receiving a read receipt from Charlie
+    # should not unblock hidden chat with Charlie for Bob.
+    assert not bob.get_chat_by_contact(bob_contact_charlie)
+
+
+def test_verified_group_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifying a member and sending a message in a group."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Hi!"
+
+    # ac1 contact is verified for ac2 because ac3 gossiped ac1 key in the "Hi!" message.
+    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
+    assert ac1_contact.get_snapshot().is_verified
+
+    # ac2 can write messages to the group.
+    snapshot.chat.send_text("Works again!")
+
+    snapshot = ac3.get_message_by_id(ac3.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_chat_messages = snapshot.chat.get_messages()
+    ac2_addr = ac2.get_config("addr")
+    assert ac1_chat_messages[-2].get_snapshot().text == f"Changed setup for {ac2_addr}"
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_verified_group_member_added_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifiying than removing and adding a member back."""
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    logging.info("ac1 creates verified group")
+    chat = ac1.create_group("Verified group", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+
+    logging.info("ac2 joins verified group")
+    qr_code, _svg = chat.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 has ac2 directly verified.
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
+
+    logging.info("ac3 joins verified group")
+    ac3_chat = ac3.secure_join(qr_code)
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
+
+    logging.info("ac2 logs in on a new device")
+    ac2 = acfactory.resetup_account(ac2)
+
+    logging.info("ac2 reverifies with ac3")
+    qr_code, _svg = ac3.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    logging.info("ac3 sends a message to the group")
+    assert len(ac3_chat.get_contacts()) == 3
+    ac3_chat.send_text("Hi!")
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("Received message %s", snapshot.text)
+    assert snapshot.text == "Hi!"
+
+    ac1.wait_for_incoming_msg_event()  # Hi!
+
+    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
+    ac3_chat.remove_contact(ac3_contact_ac2)
+    ac3_chat.add_contact(ac3_contact_ac2)
+
+    msg_id = ac2.wait_for_incoming_msg_event().msg_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert "removed" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "removed" in snapshot.text
+
+    event = ac2.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    chat_id = event.chat_id
+    message = ac2.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    logging.info("ac2 got event message: %s", snapshot.text)
+    assert "added" in snapshot.text
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert "added" in snapshot.text
+
+    chat = Chat(ac2, chat_id)
+    chat.send_text("Works again!")
+
+    msg_id = ac3.wait_for_incoming_msg_event().msg_id
+    message = ac3.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Works again!"
+
+    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
+    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
+    assert ac1_contact_ac2_snapshot.is_verified
+    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
+
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
+    """Regression test for
+    issue <https://github.com/deltachat/deltachat-core-rust/issues/4894>.
+    """
+    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
+
+    logging.info("ac3: verify with ac2")
+    qr_code, _svg = ac2.get_qr_code()
+    ac3.secure_join(qr_code)
+    ac2.wait_for_securejoin_inviter_success()
+
+    # in order for ac2 to have pending bobstate with a verified group
+    # we first create a fully joined verified group, and then start
+    # joining a second time but interrupt it, to create pending bob state
+
+    logging.info("ac1: create verified group that ac2 fully joins")
+    ch1 = ac1.create_group("Group", protect=True)
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    # ensure ac1 can write and ac2 receives messages in verified chat
+    ch1.send_text("ac1 says hello")
+    while 1:
+        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if snapshot.text == "ac1 says hello":
+            assert snapshot.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.remove()
+    logging.info("ac2 now has pending bobstate but ac1 is shutoff")
+
+    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
+    assert ac3.get_contact_by_addr(ac2.get_config("addr")).get_snapshot().is_verified
+    assert ac2.get_contact_by_addr(ac3.get_config("addr")).get_snapshot().is_verified
+
+    logging.info("ac3: create a verified group VG with ac2")
+    vg = ac3.create_group("ac3-created", protect=True)
+    vg.add_contact(ac3.get_contact_by_addr(ac2.get_config("addr")))
+
+    # ensure ac2 receives message in VG
+    vg.send_text("hello")
+    while 1:
+        msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if msg.text == "hello":
+            assert msg.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
+    qr_code, _svg = vg.get_qr_code()
+    ac4.secure_join(qr_code)
+    ac3.wait_for_securejoin_inviter_success()
+    while 1:
+        ev = ac2.wait_for_event()
+        if "added by unrelated SecureJoin" in str(ev):
+            return
+
+
+def test_qr_new_group_unblocked(acfactory):
+    """Regression test for a bug introduced in core v1.113.0.
+    ac2 scans a verified group QR code created by ac1.
+    This results in creation of a blocked 1:1 chat with ac1 on ac2,
+    but ac1 contact is not blocked on ac2.
+    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
+    ac2 should receive a message and create a contact request for the group.
+    Due to a bug previously ac2 created a blocked group.
+    """
+
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1_chat = ac1.create_group("Group for joining", protect=True)
+    qr_code, _svg = ac1_chat.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    ac1.wait_for_securejoin_inviter_success()
+
+    ac1_new_chat = ac1.create_group("Another group")
+    ac1_new_chat.add_contact(ac1.get_contact_by_addr(ac2.get_config("addr")))
+    # Receive "Member added" message.
+    ac2.wait_for_incoming_msg_event()
+
+    ac1_new_chat.send_text("Hello!")
+    ac2_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert ac2_msg.text == "Hello!"
+    assert ac2_msg.chat.get_basic_snapshot().is_contact_request
+
+
+def test_aeap_flow_verified(acfactory):
+    """Test that a new address is added to a contact when it changes its address."""
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+
+    logging.info("ac1: create verified-group QR, ac2 scans and joins")
+    chat = ac1.create_group("hello", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+    qr_code, _svg = chat.get_qr_code()
+    logging.info("ac2: start QR-code based join-group protocol")
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    logging.info("sending first message")
+    msg_out = chat.send_text("old address").get_snapshot()
+
+    logging.info("receiving first message")
+    ac2.wait_for_incoming_msg_event()  # member added message
+    msg_in_1 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert msg_in_1.text == msg_out.text
+
+    logging.info("changing email account")
+    ac1.set_config("addr", ac1new.get_config("addr"))
+    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
+    ac1.stop_io()
+    ac1.configure()
+    ac1.start_io()
+
+    logging.info("sending second message")
+    msg_out = chat.send_text("changed address").get_snapshot()
+
+    logging.info("receiving second message")
+    msg_in_2 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    msg_in_2_snapshot = msg_in_2.get_snapshot()
+    assert msg_in_2_snapshot.text == msg_out.text
+    assert msg_in_2_snapshot.chat.id == msg_in_1.chat.id
+    assert msg_in_2.get_sender_contact().get_snapshot().address == ac1new.get_config("addr")
+    assert len(msg_in_2_snapshot.chat.get_contacts()) == 2
+    assert ac1new.get_config("addr") in [
+        contact.get_snapshot().address for contact in msg_in_2_snapshot.chat.get_contacts()
+    ]
+
+
+def test_gossip_verification(acfactory) -> None:
+    alice, bob, carol = acfactory.get_online_accounts(3)
+
+    # Bob verifies Alice.
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    # Bob verifies Carol.
+    qr_code, _svg = carol.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    bob_contact_alice = bob.create_contact(alice.get_config("addr"), "Alice")
+    bob_contact_carol = bob.create_contact(carol.get_config("addr"), "Carol")
+    carol_contact_alice = carol.create_contact(alice.get_config("addr"), "Alice")
+
+    logging.info("Bob creates an Autocrypt group")
+    bob_group_chat = bob.create_group("Autocrypt Group")
+    assert not bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Autocrypt group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Autocrypt group"
+    assert snapshot.show_padlock
+
+    # Autocrypt group does not propagate verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert not carol_contact_alice_snapshot.is_verified
+
+    logging.info("Bob creates a Securejoin group")
+    bob_group_chat = bob.create_group("Securejoin Group", protect=True)
+    assert bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Securejoin group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Securejoin group"
+    assert snapshot.show_padlock
+
+    # Securejoin propagates verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert carol_contact_alice_snapshot.is_verified
+
+
+def test_securejoin_after_contact_resetup(acfactory) -> None:
+    """
+    Regression test for a bug that prevented joining verified group with a QR code
+    if the group is already created and contains
+    a contact with inconsistent (Autocrypt and verified keys exist but don't match) key state.
+    """
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    # ac3 creates protected group with ac1.
+    ac3_chat = ac3.create_group("Verified group", protect=True)
+
+    # ac1 joins ac3 group.
+    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac1.secure_join(ac3_qr_code)
+    ac1.wait_for_securejoin_joiner_success()
+
+    # ac1 waits for member added message and creates a QR code.
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+
+    # ac2 verifies ac1
+    qr_code, _svg = ac1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 is verified for ac2.
+    ac2_contact_ac1 = ac2.create_contact(ac1.get_config("addr"), "")
+    assert ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 resetups the account.
+    ac1 = acfactory.resetup_account(ac1)
+
+    # ac1 sends a message to ac2.
+    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+    ac1_chat_ac2.send_text("Hello!")
+
+    # ac2 receives a message.
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+
+    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+    assert not ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 goes offline.
+    ac1.remove()
+
+    # Scanning a QR code results in creating an unprotected group with an inviter.
+    # In this case inviter is ac1 which has an inconsistent key state.
+    # Normally inviter becomes verified as a result of Securejoin protocol
+    # and then the group chat becomes verified when "Member added" is received,
+    # but in this case ac1 is offline and this Securejoin process will never finish.
+    logging.info("ac2 scans ac1 QR code, this is not expected to finish")
+    ac2.secure_join(ac1_qr_code)
+
+    logging.info("ac2 scans ac3 QR code")
+    ac2.secure_join(ac3_qr_code)
+
+    logging.info("ac2 waits for joiner success")
+    ac2.wait_for_securejoin_joiner_success()
+
+    # Wait for member added.
+    logging.info("ac2 waits for member added message")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.is_info
+    ac2_chat = snapshot.chat
+    assert ac2_chat.get_basic_snapshot().is_protected
+    assert len(ac2_chat.get_contacts()) == 3
+
+    # ac1 is still "not verified" for ac2 due to inconsistent state.
+    assert not ac2_contact_ac1.get_snapshot().is_verified

deltachat-rpc-client/tests/test_something.py

@@ -1,4 +1,6 @@
 import concurrent.futures
+import json
+import subprocess
 from unittest.mock import MagicMock
 
 import pytest
@@ -42,7 +44,7 @@ def test_acfactory(acfactory) -> None:
     account = acfactory.new_configured_account()
     while True:
         event = account.wait_for_event()
-        if event.type == EventType.CONFIGURE_PROGRESS:
+        if event.kind == EventType.CONFIGURE_PROGRESS:
             assert event.progress != 0  # Progress 0 indicates error.
             if event.progress == 1000:  # Success
                 break
@@ -71,7 +73,7 @@ def test_account(acfactory) -> None:
 
     while True:
         event = bob.wait_for_event()
-        if event.type == EventType.INCOMING_MSG:
+        if event.kind == EventType.INCOMING_MSG:
             chat_id = event.chat_id
             msg_id = event.msg_id
             break
@@ -138,12 +140,9 @@ def test_chat(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.type == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
     assert snapshot.chat_id == chat_id
@@ -222,12 +221,9 @@ def test_message(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.type == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
 
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
@@ -263,7 +259,7 @@ def test_is_bot(acfactory) -> None:
 
     while True:
         event = bob.wait_for_event()
-        if event.type == EventType.INCOMING_MSG:
+        if event.kind == EventType.INCOMING_MSG:
             msg_id = event.msg_id
             message = bob.get_message_by_id(msg_id)
             snapshot = message.get_snapshot()
@@ -329,7 +325,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -339,10 +335,107 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
     files = list(tmp_path.glob("*.tar"))
     alice2 = acfactory.get_unconfigured_account()
     alice2.import_backup(files[0])
+
+    assert alice2.manager.get_system_info()
+
+
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
+def test_openrpc_command_line() -> None:
+    """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
+    out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
+    openrpc = json.loads(out)
+    assert "openrpc" in openrpc
+    assert "methods" in openrpc
+
+
+def test_provider_info(rpc) -> None:
+    account_id = rpc.add_account()
+
+    provider_info = rpc.get_provider_info(account_id, "example.org")
+    assert provider_info["id"] == "example.com"
+
+    provider_info = rpc.get_provider_info(account_id, "uep7oiw4ahtaizuloith.org")
+    assert provider_info is None
+
+    # Test MX record resolution.
+    provider_info = rpc.get_provider_info(account_id, "github.com")
+    assert provider_info["id"] == "gmail"
+
+    # Disable MX record resolution.
+    rpc.set_config(account_id, "socks5_enabled", "1")
+    provider_info = rpc.get_provider_info(account_id, "github.com")
+    assert provider_info is None
+
+
+def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+
+    # Bob creates chat manually so chat with Alice is accepted.
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    # Alice sends a message to Bob.
+    alice_chat_bob.send_text("Hello Bob!")
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+
+    # Bob sends a message to Alice.
+    bob_chat_alice = snapshot.chat
+    bob_chat_alice.accept()
+    bob_chat_alice.send_text("Hello Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+    # Alice reads Bob's message.
+    message.mark_seen()
+    while True:
+        event = bob.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    # Bob sends a message to Alice, it should also be encrypted.
+    bob_chat_alice.send_text("Hi Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock

deltachat-rpc-client/tests/test_webxdc.py

@@ -11,7 +11,7 @@ def test_webxdc(acfactory) -> None:
 
     while True:
         event = bob.wait_for_event()
-        if event.type == EventType.INCOMING_MSG:
+        if event.kind == EventType.INCOMING_MSG:
             bob_chat_alice = bob.get_chat_by_id(event.chat_id)
             message = bob.get_message_by_id(event.msg_id)
             break
@@ -43,3 +43,15 @@ def test_webxdc(acfactory) -> None:
     assert status_updates == [
         {"payload": "Second update", "serial": 2, "max_serial": 2},
     ]
+
+
+def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+
+    for i in range(2000):
+        message.send_webxdc_status_update({"payload": str(i)}, "description")

deltachat-rpc-client/tox.ini

@@ -11,7 +11,7 @@ setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
     RUST_MIN_STACK=8388608
 passenv =
-    DCC_NEW_TMP_EMAIL
+    CHATMAIL_DOMAIN
 deps =
     pytest
     pytest-timeout
@@ -28,4 +28,6 @@ commands =
     ruff src/ examples/ tests/
 
 [pytest]
-timeout = 60
+timeout = 300
+log_cli = true
+log_level = debug

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.124.1"
+version = "1.133.2"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -15,13 +15,13 @@ deltachat = { path = "..", default-features = false }
 
 anyhow = "1"
 env_logger = { version = "0.10.0" }
-futures-lite = "1.13.0"
+futures-lite = "2.0.0"
 log = "0.4"
-serde_json = "1.0.99"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
-tokio = { version = "1.29.1", features = ["io-std"] }
-tokio-util = "0.7.8"
-yerpc = { version = "0.5.1", features = ["anyhow_expose"] }
+tokio = { version = "1.33.0", features = ["io-std"] }
+tokio-util = "0.7.9"
+yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 
 [features]
 default = ["vendored"]

deltachat-rpc-server/README.md

@@ -32,3 +32,6 @@ languages other than Rust, for example:
 
 1. Python: https://github.com/deltachat/deltachat-core-rust/tree/master/deltachat-rpc-client/
 2. Go: https://github.com/deltachat/deltachat-rpc-client-go/
+
+Run `deltachat-rpc-server --version` to check the version of the server.
+Run `deltachat-rpc-server --openrpc` to get [OpenRPC](https://open-rpc.org/) specification of the provided JSON-RPC API.

deltachat-rpc-server/src/main.rs

@@ -10,6 +10,7 @@ use deltachat::constants::DC_VERSION_STR;
 use deltachat_jsonrpc::api::{Accounts, CommandApi};
 use futures_lite::stream::StreamExt;
 use tokio::io::{self, AsyncBufReadExt, BufReader};
+use yerpc::RpcServer as _;
 
 #[cfg(target_family = "unix")]
 use tokio::signal::unix as signal_unix;
@@ -39,6 +40,12 @@ async fn main_impl() -> Result<()> {
             }
             eprintln!("{}", &*DC_VERSION_STR);
             return Ok(());
+        } else if first_arg.to_str() == Some("--openrpc") {
+            if let Some(arg) = args.next() {
+                return Err(anyhow!("Unrecognized argument {:?}", arg));
+            }
+            println!("{}", CommandApi::openrpc_specification()?);
+            return Ok(());
         } else {
             return Err(anyhow!("Unrecognized option {:?}", first_arg));
         }
@@ -56,7 +63,8 @@ async fn main_impl() -> Result<()> {
 
     let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "accounts".to_string());
     log::info!("Starting with accounts directory `{}`.", path);
-    let accounts = Accounts::new(PathBuf::from(&path)).await?;
+    let writable = true;
+    let accounts = Accounts::new(PathBuf::from(&path), writable).await?;
 
     log::info!("Creating JSON-RPC API.");
     let accounts = Arc::new(RwLock::new(accounts));

deny.toml

@@ -3,6 +3,13 @@ unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
+
+    # Timing attack on RSA.
+    # Delta Chat does not use RSA for new keys
+    # and this requires precise measurement of the decryption time by the attacker.
+    # There is no fix at the time of writing this (2023-11-28).
+    # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
+    "RUSTSEC-2023-0071",
 ]
 
 [bans]
@@ -11,7 +18,7 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
-     { name = "ahash", version = "0.7.6" },
+     { name = "async-channel", version = "1.9.0" },
      { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
      { name = "bitflags", version = "1.3.2" },
@@ -25,41 +32,42 @@ skip = [
      { name = "digest", version = "<0.10" },
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
+     { name = "event-listener", version = "2.5.3" },
      { name = "getrandom", version = "<0.2" },
-     { name = "hashbrown", version = "<0.14.0" },
-     { name = "idna", version = "<0.3" },
-     { name = "indexmap", version = "<2.0.0" },
-     { name = "linux-raw-sys", version = "0.3.8" },
-     { name = "num-derive", version = "0.3.3" },
+     { name = "h2", version = "0.3.22" },
+     { name = "http-body", version = "0.4.5" },
+     { name = "http", version = "0.2.11" },
+     { name = "hyper", version = "0.14.27" },
+     { name = "idna", version = "0.4.0" },
      { name = "pem-rfc7468", version = "0.6.0" },
      { name = "pkcs8", version = "0.9.0" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
-     { name = "redox_syscall", version = "0.2.16" },
+     { name = "redox_syscall", version = "0.3.5" },
+     { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
-     { name = "rustix", version = "0.37.21" },
+     { name = "ring", version = "0.16.20" },
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },
-     { name = "socket2", version = "0.4.9" },
      { name = "spin", version = "<0.9.6" },
      { name = "spki", version = "0.6.0" },
      { name = "syn", version = "1.0.109" },
      { name = "time", version = "<0.3" },
+     { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
-     { name = "windows_aarch64_gnullvm", version = "<0.48" },
-     { name = "windows_aarch64_msvc", version = "<0.48" },
-     { name = "windows_i686_gnu", version = "<0.48" },
-     { name = "windows_i686_msvc", version = "<0.48" },
-     { name = "windows-sys", version = "<0.48" },
-     { name = "windows-targets", version = "<0.48" },
-     { name = "windows_x86_64_gnullvm", version = "<0.48" },
+     { name = "windows_aarch64_gnullvm", version = "<0.52" },
+     { name = "windows_aarch64_msvc", version = "<0.52" },
+     { name = "windows_i686_gnu", version = "<0.52" },
+     { name = "windows_i686_msvc", version = "<0.52" },
+     { name = "windows-sys", version = "<0.52" },
+     { name = "windows-targets", version = "<0.52" },
      { name = "windows", version = "0.32.0" },
-     { name = "windows_x86_64_gnu", version = "<0.48" },
-     { name = "windows_x86_64_msvc", version = "<0.48" },
-     { name = "winreg", version = "0.10.1" },
+     { name = "windows_x86_64_gnullvm", version = "<0.52" },
+     { name = "windows_x86_64_gnu", version = "<0.52" },
+     { name = "windows_x86_64_msvc", version = "<0.52" },
 ]
 
 
@@ -91,5 +99,5 @@ license-files = [
 github = [
        "async-email",
        "deltachat",
-       "quinn-rs",
+       "djc",
 ]

draft/aeap-mvp.md

@@ -57,7 +57,7 @@ Note that usually a mail is signed by a key that has a UID matching the from add
 ### Notes:
   
 - We treat protected and non-protected chats the same
-- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more pepole know what old addresses you had).
+- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more people know what old addresses you had).
 - As soon as we encrypt read receipts, sending a read receipt will be enough to tell a lot of people that you transitioned
 - AEAP will make the problem of inconsistent group state worse, both because it doesn't work if the message is unencrypted (even if the design allowed it, it would be problematic security-wise) and because some chat partners may have gotten the transition and some not. We should do something against this at some point in the future, like asking the user whether they want to add/remove the members to restore consistent group state.
 
@@ -108,7 +108,7 @@ The most obvious alternative would be to create a new contact with the new addre
 #### Upsides:
 - With this approach, it's easier to switch to a model where the info about the transition is encoded in the PGP key. Since the key is gossiped, the information about the transition will spread virally.
 - (Also, less important: Slightly faster transition: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.)
-- It's easier to implement (if too many problems turn up, we can still switch to another approach and didn't wast that much development time.)
+- It's easier to implement (if too many problems turn up, we can still switch to another approach and didn't waste that much development time.)
 
 [full messages](https://github.com/deltachat/deltachat-core-rust/pull/2896#discussion_r852002161)
   

examples/simple.rs

@@ -1,100 +0,0 @@
-use deltachat::chat::{self, ChatId};
-use deltachat::chatlist::*;
-use deltachat::config;
-use deltachat::contact::*;
-use deltachat::context::*;
-use deltachat::message::Message;
-use deltachat::stock_str::StockStrings;
-use deltachat::{EventType, Events};
-use tempfile::tempdir;
-
-fn cb(event: EventType) {
-    match event {
-        EventType::ConfigureProgress { progress, .. } => {
-            log::info!("progress: {}", progress);
-        }
-        EventType::Info(msg) => {
-            log::info!("{}", msg);
-        }
-        EventType::Warning(msg) => {
-            log::warn!("{}", msg);
-        }
-        EventType::Error(msg) => {
-            log::error!("{}", msg);
-        }
-        event => {
-            log::info!("{:?}", event);
-        }
-    }
-}
-
-/// Run with `RUST_LOG=simple=info cargo run --release --example simple -- email pw`.
-#[tokio::main]
-async fn main() {
-    pretty_env_logger::try_init_timed().ok();
-
-    let dir = tempdir().unwrap();
-    let dbfile = dir.path().join("db.sqlite");
-    log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(&dbfile, 0, Events::new(), StockStrings::new())
-        .await
-        .expect("Failed to create context");
-    let info = ctx.get_info().await;
-    log::info!("info: {:#?}", info);
-
-    let events = ctx.get_event_emitter();
-    let events_spawn = tokio::task::spawn(async move {
-        while let Some(event) = events.recv().await {
-            cb(event.typ);
-        }
-    });
-
-    log::info!("configuring");
-    let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 3, "requires email password");
-    let email = args[1].clone();
-    let pw = args[2].clone();
-    ctx.set_config(config::Config::Addr, Some(&email))
-        .await
-        .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw))
-        .await
-        .unwrap();
-
-    ctx.configure().await.unwrap();
-
-    log::info!("------ RUN ------");
-    ctx.start_io().await;
-    log::info!("--- SENDING A MESSAGE ---");
-
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
-        .await
-        .unwrap();
-    let chat_id = ChatId::create_for_contact(&ctx, contact_id).await.unwrap();
-
-    for i in 0..1 {
-        log::info!("sending message {}", i);
-        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {i}nth message!"))
-            .await
-            .unwrap();
-    }
-
-    // wait for the message to be sent out
-    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
-
-    log::info!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
-
-    for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
-            .await
-            .unwrap();
-        log::info!("[{}] msg: {:?}", i, msg);
-    }
-
-    log::info!("stopping");
-    ctx.stop_io().await;
-    log::info!("closing");
-    drop(ctx);
-    events_spawn.await.unwrap();
-}

node/README.md

@@ -40,7 +40,7 @@ npm install deltachat-node
 
 ## Dependencies
 
-- Nodejs >= `v16.0.0`
+- Nodejs >= `v18.0.0`
 - rustup (optional if you can't use the prebuilds)
 
 > On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
@@ -113,8 +113,8 @@ Then, in the `deltachat-desktop` repository, run:
 deltachat doesn't support universal (fat) binaries (that contain builds for both cpu architectures) yet, until it does you can use the following workaround to get x86_64 builds:
 
 ```
-$ fnm install 17 --arch x64
-$ fnm use 17
+$ fnm install 19 --arch x64
+$ fnm use 19
 $ node -p process.arch
 # result should be x64
 $ rustup target add x86_64-apple-darwin
@@ -127,8 +127,8 @@ $ npm run test
 If your node and electron are already build for arm64 you can also try building for arm:
 
 ```
-$ fnm install 16 --arch arm64
-$ fnm use 16
+$ fnm install 18 --arch arm64
+$ fnm use 18
 $ node -p process.arch
 # result should be arm64
 $ npm_config_arch=arm64 npm run build
@@ -204,10 +204,10 @@ Running `npm test` ends with showing a code coverage report, which is produced b
 
 The coverage report from `nyc` in the console is rather limited. To get a more detailed coverage report you can run `npm run coverage-html-report`. This will produce a html report from the `nyc` data and display it in a browser on your local machine.
 
-To run the integration tests you need to set the `DCC_NEW_TMP_EMAIL` environment variables. E.g.:
+To run the integration tests you need to set the `CHATMAIL_DOMAIN` environment variables. E.g.:
 
 ```
-$ export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=[token]
+$ export CHATMAIL_DOMAIN=chat.example.org
 $ npm run test
 ```
 

node/constants.js

@@ -28,9 +28,11 @@ module.exports = {
   DC_DOWNLOAD_DONE: 0,
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
+  DC_EVENT_CONFIG_SYNCED: 2111,
   DC_EVENT_CONNECTIVITY_CHANGED: 2100,
   DC_EVENT_CONTACTS_CHANGED: 2030,
   DC_EVENT_DELETED_BLOB_FILE: 151,
@@ -78,6 +80,7 @@ module.exports = {
   DC_INFO_EPHEMERAL_TIMER_CHANGED: 10,
   DC_INFO_GROUP_IMAGE_CHANGED: 3,
   DC_INFO_GROUP_NAME_CHANGED: 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL: 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED: 8,
   DC_INFO_LOCATION_ONLY: 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP: 4,
@@ -159,6 +162,8 @@ module.exports = {
   DC_STR_BROADCAST_LIST: 115,
   DC_STR_CANNOT_LOGIN: 60,
   DC_STR_CANTDECRYPT_MSG_BODY: 29,
+  DC_STR_CHAT_PROTECTION_DISABLED: 171,
+  DC_STR_CHAT_PROTECTION_ENABLED: 170,
   DC_STR_CONFIGURATION_FAILED: 84,
   DC_STR_CONNECTED: 107,
   DC_STR_CONNTECTING: 108,
@@ -222,11 +227,13 @@ module.exports = {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU: 124,
   DC_STR_IMAGE: 9,
   DC_STR_INCOMING_MESSAGES: 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL: 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY: 111,
   DC_STR_LOCATION: 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER: 137,
   DC_STR_LOCATION_ENABLED_BY_YOU: 136,
   DC_STR_MESSAGES: 114,
+  DC_STR_MESSAGE_ADD_MEMBER: 173,
   DC_STR_MSGACTIONBYME: 63,
   DC_STR_MSGACTIONBYUSER: 62,
   DC_STR_MSGADDMEMBER: 17,
@@ -237,6 +244,7 @@ module.exports = {
   DC_STR_MSGGRPNAME: 15,
   DC_STR_MSGLOCATIONDISABLED: 65,
   DC_STR_MSGLOCATIONENABLED: 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE: 172,
   DC_STR_NOMESSAGES: 1,
   DC_STR_NOT_CONNECTED: 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER: 113,
@@ -244,12 +252,6 @@ module.exports = {
   DC_STR_OUTGOING_MESSAGES: 104,
   DC_STR_PARTIAL_DOWNLOAD_MSG_BODY: 99,
   DC_STR_PART_OF_TOTAL_USED: 116,
-  DC_STR_PROTECTION_DISABLED: 89,
-  DC_STR_PROTECTION_DISABLED_BY_OTHER: 161,
-  DC_STR_PROTECTION_DISABLED_BY_YOU: 160,
-  DC_STR_PROTECTION_ENABLED: 88,
-  DC_STR_PROTECTION_ENABLED_BY_OTHER: 159,
-  DC_STR_PROTECTION_ENABLED_BY_YOU: 158,
   DC_STR_QUOTA_EXCEEDING_MSG_BODY: 98,
   DC_STR_READRCPT: 31,
   DC_STR_READRCPT_MAILBODY: 32,

node/events.js

@@ -34,6 +34,7 @@ module.exports = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
 }

node/lib/constants.ts

@@ -28,9 +28,11 @@ export enum C {
   DC_DOWNLOAD_DONE = 0,
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
+  DC_EVENT_CONFIG_SYNCED = 2111,
   DC_EVENT_CONNECTIVITY_CHANGED = 2100,
   DC_EVENT_CONTACTS_CHANGED = 2030,
   DC_EVENT_DELETED_BLOB_FILE = 151,
@@ -78,6 +80,7 @@ export enum C {
   DC_INFO_EPHEMERAL_TIMER_CHANGED = 10,
   DC_INFO_GROUP_IMAGE_CHANGED = 3,
   DC_INFO_GROUP_NAME_CHANGED = 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL = 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED = 8,
   DC_INFO_LOCATION_ONLY = 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP = 4,
@@ -159,6 +162,8 @@ export enum C {
   DC_STR_BROADCAST_LIST = 115,
   DC_STR_CANNOT_LOGIN = 60,
   DC_STR_CANTDECRYPT_MSG_BODY = 29,
+  DC_STR_CHAT_PROTECTION_DISABLED = 171,
+  DC_STR_CHAT_PROTECTION_ENABLED = 170,
   DC_STR_CONFIGURATION_FAILED = 84,
   DC_STR_CONNECTED = 107,
   DC_STR_CONNTECTING = 108,
@@ -222,11 +227,13 @@ export enum C {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU = 124,
   DC_STR_IMAGE = 9,
   DC_STR_INCOMING_MESSAGES = 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL = 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY = 111,
   DC_STR_LOCATION = 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER = 137,
   DC_STR_LOCATION_ENABLED_BY_YOU = 136,
   DC_STR_MESSAGES = 114,
+  DC_STR_MESSAGE_ADD_MEMBER = 173,
   DC_STR_MSGACTIONBYME = 63,
   DC_STR_MSGACTIONBYUSER = 62,
   DC_STR_MSGADDMEMBER = 17,
@@ -237,6 +244,7 @@ export enum C {
   DC_STR_MSGGRPNAME = 15,
   DC_STR_MSGLOCATIONDISABLED = 65,
   DC_STR_MSGLOCATIONENABLED = 64,
+  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE = 172,
   DC_STR_NOMESSAGES = 1,
   DC_STR_NOT_CONNECTED = 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER = 113,
@@ -244,12 +252,6 @@ export enum C {
   DC_STR_OUTGOING_MESSAGES = 104,
   DC_STR_PARTIAL_DOWNLOAD_MSG_BODY = 99,
   DC_STR_PART_OF_TOTAL_USED = 116,
-  DC_STR_PROTECTION_DISABLED = 89,
-  DC_STR_PROTECTION_DISABLED_BY_OTHER = 161,
-  DC_STR_PROTECTION_DISABLED_BY_YOU = 160,
-  DC_STR_PROTECTION_ENABLED = 88,
-  DC_STR_PROTECTION_ENABLED_BY_OTHER = 159,
-  DC_STR_PROTECTION_ENABLED_BY_YOU = 158,
   DC_STR_QUOTA_EXCEEDING_MSG_BODY = 98,
   DC_STR_READRCPT = 31,
   DC_STR_READRCPT_MAILBODY = 32,
@@ -321,6 +323,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
 }

node/lib/context.ts

@@ -36,7 +36,7 @@ export class Context extends EventEmitter {
     }
   }
 
-  /** Opens a stanalone context (without an account manager)
+  /** Opens a standalone context (without an account manager)
    * automatically starts the event handler */
   static open(cwd: string): Context {
     const dbFile = join(cwd, 'db.sqlite')
@@ -699,23 +699,6 @@ export class Context extends EventEmitter {
     )
   }
 
-  /**
-   *
-   * @param chatId
-   * @param protect
-   * @returns success boolean
-   */
-  setChatProtection(chatId: number, protect: boolean) {
-    debug(`setChatProtection ${chatId} ${protect}`)
-    return Boolean(
-      binding.dcn_set_chat_protection(
-        this.dcn_context,
-        Number(chatId),
-        protect ? 1 : 0
-      )
-    )
-  }
-
   getChatEphemeralTimer(chatId: number): number {
     debug(`getChatEphemeralTimer ${chatId}`)
     return binding.dcn_get_chat_ephemeral_timer(

node/lib/deltachat.ts

@@ -21,12 +21,15 @@ export class AccountManager extends EventEmitter {
   accountDir: string
   jsonRpcStarted = false
 
-  constructor(cwd: string, os = 'deltachat-node') {
+  constructor(cwd: string, writable = true) {
     super()
     debug('DeltaChat constructor')
 
     this.accountDir = cwd
-    this.dcn_accounts = binding.dcn_accounts_new(os, this.accountDir)
+    this.dcn_accounts = binding.dcn_accounts_new(
+      this.accountDir,
+      writable ? 1 : 0
+    )
   }
 
   getAllAccountIds() {
@@ -175,7 +178,7 @@ export class AccountManager extends EventEmitter {
   static newTemporary() {
     let directory = null
     while (true) {
-      const randomString = Math.random().toString(36).substr(2, 5)
+      const randomString = Math.random().toString(36).substring(2, 5)
       directory = join(tmpdir(), 'deltachat-' + randomString)
       if (!existsSync(directory)) break
     }

node/src/module.c

@@ -1399,18 +1399,6 @@ NAPI_METHOD(dcn_set_chat_name) {
   NAPI_RETURN_INT32(result);
 }
 
-NAPI_METHOD(dcn_set_chat_protection) {
-  NAPI_ARGV(3);
-  NAPI_DCN_CONTEXT();
-  NAPI_ARGV_UINT32(chat_id, 1);
-  NAPI_ARGV_INT32(protect, 1);
-
-  int result = dc_set_chat_protection(dcn_context->dc_context,
-                                      chat_id,
-                                      protect);
-  NAPI_RETURN_INT32(result);
-}
-
 NAPI_METHOD(dcn_get_chat_ephemeral_timer) {
   NAPI_ARGV(2);
   NAPI_DCN_CONTEXT();
@@ -2915,8 +2903,8 @@ NAPI_METHOD(dcn_msg_get_webxdc_blob){
 
 NAPI_METHOD(dcn_accounts_new) {
   NAPI_ARGV(2);
-  NAPI_ARGV_UTF8_MALLOC(os_name, 0);
-  NAPI_ARGV_UTF8_MALLOC(dir, 1);
+  NAPI_ARGV_UTF8_MALLOC(dir, 0);
+  NAPI_ARGV_INT32(writable, 1);
   TRACE("calling..");
 
   dcn_accounts_t* dcn_accounts = calloc(1, sizeof(dcn_accounts_t));
@@ -2925,7 +2913,7 @@ NAPI_METHOD(dcn_accounts_new) {
   }
 
 
-  dcn_accounts->dc_accounts = dc_accounts_new(os_name, dir);
+  dcn_accounts->dc_accounts = dc_accounts_new(dir, writable);
 
   napi_value result;
   NAPI_STATUS_THROWS(napi_create_external(env, dcn_accounts,
@@ -3491,7 +3479,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_send_msg);
   NAPI_EXPORT_FUNCTION(dcn_send_videochat_invitation);
   NAPI_EXPORT_FUNCTION(dcn_set_chat_name);
-  NAPI_EXPORT_FUNCTION(dcn_set_chat_protection);
   NAPI_EXPORT_FUNCTION(dcn_get_chat_ephemeral_timer);
   NAPI_EXPORT_FUNCTION(dcn_set_chat_ephemeral_timer);
   NAPI_EXPORT_FUNCTION(dcn_set_chat_profile_image);

node/test/test.mjs

@@ -1,33 +1,28 @@
 // @ts-check
-import DeltaChat from '../dist'
+import { DeltaChat } from '../dist/index.js'
 
 import { deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
-import { EventId2EventName, C } from '../dist/constants'
+import { EventId2EventName, C } from '../dist/constants.js'
 import { join } from 'path'
 import { statSync } from 'fs'
-import { Context } from '../dist/context'
-import fetch from 'node-fetch'
+import { Context } from '../dist/context.js'
+import {fileURLToPath} from 'url';
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
-async function createTempUser(url) {
-  async function postData(url = '') {
-    // Default options are marked with *
-    const response = await fetch(url, {
-      method: 'POST', // *GET, POST, PUT, DELETE, etc.
-      headers: {
-        'cache-control': 'no-cache',
-      },
-    })
-    if (!response.ok) {
-      throw new Error('request failed: ' + response.body.read())
-    }
-    return response.json() // parses JSON response into native JavaScript objects
+function createTempUser(chatmailDomain) {
+  const charset = "2345789acdefghjkmnpqrstuvwxyz";
+  let user = "ci-";
+  for (let i = 0; i < 6; i++) {
+    user += charset[Math.floor(Math.random() * charset.length)];
   }
-
-  return await postData(url)
+  const email = user + "@" + chatmailDomain;
+  return { email: email, password: user + "$" + user };
 }
 
 describe('static tests', function () {
@@ -270,7 +265,7 @@ describe('Basic offline Tests', function () {
       'quota_exceeding',
       'scan_all_folders_debounce_secs',
       'selfavatar',
-      'send_sync_msgs',
+      'sync_msgs',
       'sentbox_watch',
       'show_emails',
       'socks5_enabled',
@@ -676,9 +671,9 @@ describe('Offline Tests with unconfigured account', function () {
 
     const lot = chatList.getSummary(0)
     strictEqual(lot.getId(), 0, 'lot has no id')
-    strictEqual(lot.getState(), C.DC_STATE_UNDEFINED, 'correct state')
+    strictEqual(lot.getState(), C.DC_STATE_IN_NOTICED, 'correct state')
 
-    const text = 'No messages.'
+    const text = 'Others will only see this group after you sent a first message.'
     context.createGroupChat('groupchat1111')
     chatList = context.getChatList(0, 'groupchat1111', null)
     strictEqual(
@@ -768,14 +763,7 @@ describe('Integration tests', function () {
   })
 
   this.beforeAll(async function () {
-    if (!process.env.DCC_NEW_TMP_EMAIL) {
-      console.log(
-        'Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests'
-      )
-      this.skip()
-    }
-
-    account = await createTempUser(process.env.DCC_NEW_TMP_EMAIL)
+    account = createTempUser(process.env.CHATMAIL_DOMAIN)
     if (!account || !account.email || !account.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"

node/tsconfig.json

@@ -13,10 +13,10 @@
     },
     "exclude": ["node_modules", "deltachat-core-rust", "dist", "scripts"],
     "typedocOptions": {
-        "mode": "file",
         "out": "docs",
-        "excludeNotExported": true,
+        "excludePrivate": true,
         "defaultCategory": "index",
-        "includeVersion": true
+        "includeVersion": true,
+        "entryPoints": ["lib/index.ts"]
     }
 }

node/windows.md

@@ -6,7 +6,7 @@ E.g via <https://git-scm.com/download/win>
 
 ## install node
 
-Download and install `v16` from <https://nodejs.org/en/>
+Download and install `v18` from <https://nodejs.org/en/>
 
 ## install rust
 

package.json

@@ -2,28 +2,24 @@
   "dependencies": {
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
-    "node-gyp-build": "^4.1.0"
+    "node-gyp-build": "^4.6.1"
   },
   "description": "node.js bindings for deltachat-core",
   "devDependencies": {
     "@types/debug": "^4.1.7",
-    "@types/node": "^16.11.26",
-    "chai": "^4.2.0",
+    "@types/node": "^20.8.10",
+    "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
-    "esm": "^3.2.25",
-    "hallmark": "^2.0.0",
     "mocha": "^8.2.1",
-    "node-fetch": "^2.6.7",
-    "node-gyp": "^9.0.0",
-    "opn-cli": "^5.0.0",
-    "prebuildify": "^3.0.0",
-    "prebuildify-ci": "^1.0.4",
-    "prettier": "^2.0.5",
-    "typedoc": "^0.17.0",
-    "typescript": "^3.9.10"
+    "node-gyp": "^10.0.0",
+    "prebuildify": "^5.0.1",
+    "prebuildify-ci": "^1.0.5",
+    "prettier": "^3.0.3",
+    "typedoc": "^0.25.3",
+    "typescript": "^5.2.2"
   },
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=18.0.0"
   },
   "files": [
     "node/scripts/*",
@@ -49,16 +45,15 @@
     "build:core:rust": "node node/scripts/rebuild-core.js",
     "clean": "rm -rf node/dist node/build node/prebuilds node/node_modules ./target",
     "download-prebuilds": "prebuildify-ci download",
-    "hallmark": "hallmark --fix",
     "install": "node node/scripts/install.js",
     "install:prebuilds": "cd node && node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"",
     "lint": "prettier --check \"node/lib/**/*.{ts,tsx}\"",
     "lint-fix": "prettier --write \"node/lib/**/*.{ts,tsx}\" \"node/test/**/*.js\"",
-    "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
+    "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
+    "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.124.1"
+  "version": "1.133.2"
 }

python/README.rst

@@ -1,6 +1,6 @@
-=========================
-DeltaChat Python bindings
-=========================
+============================
+CFFI Python Bindings
+============================
 
 This package provides `Python bindings`_ to the `deltachat-core library`_
 which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
@@ -8,157 +8,3 @@ a low-level Chat/Contact/Message API to user interfaces and bots.
 
 .. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
 .. _`Python bindings`: https://py.delta.chat/
-
-Installing pre-built packages (Linux-only)
-==========================================
-
-If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself`__.
-
-__ sourceinstall_
-
-We recommend to first create a fresh Python virtual environment
-and activate it in your shell::
-
-    python -m venv env
-    source env/bin/activate
-
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``env`` directory and leaves
-your system installation alone.
-
-For Linux we build wheels for all releases and push them to a python package
-index. To install the latest release::
-
-    pip install deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
-
-Running tests
-=============
-
-Recommended way to run tests is using `scripts/run-python-test.sh`
-script provided in the core repository.
-
-This script compiles the library in debug mode and runs the tests using `tox`_.
-By default it will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _`tox`: https://tox.wiki
-.. _livetests:
-
-Running "live" tests with temporary accounts
---------------------------------------------
-
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
-
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
-
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
-These accounts are removed automatically as they expire.
-After setting the variable, either rerun `scripts/run-python-test.sh`
-or run offline and online tests with `tox` directly::
-
-    tox -e py
-
-Each test run creates new accounts.
-
-Developing the bindings
------------------------
-
-If you want to develop or debug the bindings,
-you can create a testing development environment using `tox`::
-
-    export DCC_RS_DEV="$PWD"
-    export DCC_RS_TARGET=debug
-    tox -c python --devenv env -e py
-    . env/bin/activate
-
-Inside this environment the bindings are installed
-in editable mode (as if installed with `python -m pip install -e`)
-together with the testing dependencies like `pytest` and its plugins.
-
-You can then edit the source code in the development tree
-and quickly run `pytest` manually without waiting  for `tox`
-to recreating the virtual environment each time.
-
-.. _sourceinstall:
-
-Installing bindings from source
-===============================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-First, build the core library::
-
-   cargo build --release -p deltachat_ffi --features jsonrpc
-
-`jsonrpc` feature is required even if not used by the bindings
-because `deltachat.h` includes JSON-RPC functions unconditionally.
-
-Create the virtual environment and activate it:
-
-   python -m venv env
-   source env/bin/activate
-
-Build and install the bindings:
-
-   export DCC_RS_DEV="$PWD"
-   export DCC_RS_TARGET=release
-   python -m pip install ./python
-
-`DCC_RS_DEV` environment variable specifies the location of
-the core development tree. If this variable is not set,
-`libdeltachat` library and `deltachat.h` header are expected
-to be installed system-wide.
-
-When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
-the build profile name to look up the artifacts
-in the target directory.
-In this case setting it can be skipped because
-`DCC_RS_TARGET=release` is the default.
-
-Building manylinux based wheels
-===============================
-
-Building portable manylinux wheels which come with libdeltachat.so
-can be done with Docker_ or Podman_.
-
-.. _Docker: https://www.docker.com/
-.. _Podman: https://podman.io/
-
-If you want to build your own wheels, build container image first::
-
-   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
-   $ docker build -t deltachat/coredeps scripts/coredeps
-
-This will use the ``scripts/coredeps/Dockerfile`` to build
-container image called ``deltachat/coredeps``.  You can afterwards
-find it with::
-
-   $ docker images
-
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v $(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh

python/doc/Makefile

@@ -1,197 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-VERSION       = $(shell python -c "import conf ; print(conf.version)")
-DOCZIP        = devpi-$(VERSION).doc.zip
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = _build
-RSYNCOPTS     = -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-
-export HOME=/tmp/home
-export TESTHOME=$(HOME)
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# This variable is not auto generated as the order is important.
-USER_MAN_CHAPTERS = commands\
-                    user\
-                    indices\
-                    packages\
-#                 userman/index.rst\
-#                 userman/devpi_misc.rst\
-#                 userman/devpi_concepts.rst\
-
-
-#export DEVPI_CLIENTDIR=$(CURDIR)/.tmp_devpi_user_man/client
-#export DEVPI_SERVERDIR=$(CURDIR)/.tmp_devpi_user_man/server
-
-chapter = commands
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \
-        epub latex latexpdf text man changes linkcheck doctest gettext install \
-        quickstart-releaseprocess quickstart-pypimirror quickstart-server regen \
-        prepare-quickstart\
-        regen.server-fresh regen.server-restart regen.server-clean\
-        regen.uman-all regen.uman
-
-help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  dirhtml    to make HTML files named index.html in directories"
-	@echo "  singlehtml to make a single large HTML file"
-	@echo "  pickle     to make pickle files"
-	@echo "  json       to make JSON files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
-	@echo "  qthelp     to make HTML files and a qthelp project"
-	@echo "  devhelp    to make HTML files and a Devhelp project"
-	@echo "  epub       to make an epub"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
-	@echo "  texinfo    to make Texinfo files"
-	@echo "  info       to make Texinfo files and run them through makeinfo"
-	@echo "  gettext    to make PO message catalogs"
-	@echo "  changes    to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck  to check all external links for integrity"
-	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
-	@echo
-	@echo "User Manual Regen Targets"
-	@echo "  regen.uman        regenerates page. of the user manual chapeter e.g. regen.uman chapter=..."
-	@echo "  regen.uman-all    regenerates the user manual"
-	@echo "  regen.uman-clean  stop temp server and clean up directory"
-	@echo "  Chapter List: $(USER_MAN_CHAPTERS)"
-
-clean:
-	-rm -rf $(BUILDDIR)/*
-
-version:
-	@echo "version $(VERSION)"
-
-doczip: html
-	python doczip.py $(DOCZIP) _build/html
-
-install: html
-	rsync -avz $(RSYNCOPTS) _build/html/ delta@py.delta.chat:build/master
-
-
-html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
-	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/devpi.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/devpi.qhc"
-
-devhelp:
-	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
-	@echo
-	@echo "Build finished."
-	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/devpi"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/devpi"
-	@echo "# devhelp"
-
-epub:
-	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
-	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make' in that directory to run these through (pdf)latex" \
-	      "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through pdflatex..."
-	$(MAKE) -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
-	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
-	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
-	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
-	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo
-	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
-	@echo "Run \`make' in that directory to run these through makeinfo" \
-	      "(use \`make info' here to do that automatically)."
-
-info:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo "Running Texinfo files through makeinfo..."
-	make -C $(BUILDDIR)/texinfo info
-	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
-	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
-	@echo
-	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
-

python/doc/_templates/globaltoc.html

@@ -1,17 +0,0 @@
-
-<div class="globaltoc">
-
-<ul>
-  <li><a href="{{ pathto('index') }}">index</a></li>
-  <li><a href="{{ pathto('install') }}">install</a></li>
-  <li><a href="{{ pathto('api') }}">high level API</a></li>
-  <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-</ul>
-<b>external links:</b>
-<ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
-  <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
-  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
-</ul>
-
-</div>

python/doc/_templates/sidebarintro.html

@@ -1 +0,0 @@
-<h3>deltachat {{release}}</h3>

python/doc/cffi/api.rst

@@ -1,5 +1,4 @@
-
-high level API reference
+High Level API Reference
 ========================
 
 - :class:`deltachat.Account` (your main entry point, creates the
@@ -8,28 +7,14 @@ high level API reference
 - :class:`deltachat.Chat`
 - :class:`deltachat.Message`
 
-Account
--------
-
 .. autoclass:: deltachat.Account
-    :members:
-
-
-Contact
--------
+   :members:
 
 .. autoclass:: deltachat.Contact
-    :members:
-
-Chat
-----
+   :members:
 
 .. autoclass:: deltachat.Chat
-    :members:
-
-Message
--------
+   :members:
 
 .. autoclass:: deltachat.Message
-    :members:
-
+   :members:

python/doc/cffi/examples.rst

@@ -1,11 +1,10 @@
-
-examples
+Examples
 ========
 
 Once you have :doc:`installed deltachat bindings <install>`
 you need email/password credentials for an IMAP/SMTP account.
 Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+temporary email accounts on `testrun.org <https://testrun.org/>`_ for testing.
 
 Receiving a Chat message from the command line
 ----------------------------------------------
@@ -16,11 +15,11 @@ Here is a simple bot that:
 
 - terminates the bot if the message `/quit` is sent
 
-.. include:: ../examples/echo_and_quit.py
+.. include:: ../../examples/echo_and_quit.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     $ cd examples
@@ -40,11 +39,11 @@ Here is a simple bot that:
 
 - tracks member additions and removals for all chat groups
 
-.. include:: ../examples/group_tracking.py
+.. include:: ../../examples/group_tracking.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db

python/doc/cffi/install.rst

@@ -0,0 +1,80 @@
+Install
+=======
+
+Installing pre-built packages (Linux-only)
+------------------------------------------
+
+If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
+
+We recommend to first create a fresh Python virtual environment
+and activate it in your shell::
+
+    python -m venv env
+    source env/bin/activate
+
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``env`` directory and leaves
+your system installation alone.
+
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
+
+    pip install deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
+
+.. _sourceinstall:
+
+Installing bindings from source
+-------------------------------
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+First, build the core library::
+
+   cargo build --release -p deltachat_ffi --features jsonrpc
+
+`jsonrpc` feature is required even if not used by the bindings
+because `deltachat.h` includes JSON-RPC functions unconditionally.
+
+Create the virtual environment and activate it::
+
+   python -m venv env
+   source env/bin/activate
+
+Build and install the bindings::
+
+   export DCC_RS_DEV="$PWD"
+   export DCC_RS_TARGET=release
+   python -m pip install ./python
+
+`DCC_RS_DEV` environment variable specifies the location of
+the core development tree. If this variable is not set,
+`libdeltachat` library and `deltachat.h` header are expected
+to be installed system-wide.
+
+When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
+the build profile name to look up the artifacts
+in the target directory.
+In this case setting it can be skipped because
+`DCC_RS_TARGET=release` is the default.

python/doc/cffi/intro.rst

@@ -0,0 +1,11 @@
+Introduction
+============
+
+CFFI bindings are available via the `deltachat <https://pypi.org/project/deltachat/>`_ Python package.
+The package contains both the Python bindings and the Delta Chat core.
+It is provided only for Linux.
+
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
+low-level CFFI bindings to the C interface of the Delta Chat core
+and high-level Python bindings built on top of CFFI bindings.

python/doc/cffi/lapi.rst

@@ -1,8 +1,7 @@
+Low Level API Reference
+=======================
 
-low level API reference
-===================================
-
-for full doxygen-generated C-docs, defines and functions please checkout
+For full doxygen-generated C-docs, defines and functions please checkout
 
 https://c.delta.chat
 

python/doc/cffi/manylinux.rst

@@ -0,0 +1,25 @@
+Building Manylinux-Based Wheels
+===============================
+
+Building portable manylinux wheels which come with libdeltachat.so
+can be done with Docker_ or Podman_.
+
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
+
+If you want to build your own wheels, build container image first::
+
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
+
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
+find it with::
+
+   $ docker images
+
+This docker image can be used to run tests and build Python wheels for all interpreters::
+
+    $ docker run -e CHATMAIL_DOMAIN \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/doc/cffi/tests.rst

@@ -0,0 +1,49 @@
+Running Tests
+=============
+
+Recommended way to run tests is using `scripts/run-python-test.sh`
+script provided in the core repository.
+
+This script compiles the library in debug mode and runs the tests using `tox`_.
+By default it will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real email servers.
+
+.. _`tox`: https://tox.wiki
+.. _livetests:
+
+Running "Live" Tests With Temporary Accounts
+--------------------------------------------
+
+If you want to run live functional tests
+you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
+that creates email accounts like this::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+With this account-creation setting, pytest runs create ephemeral email accounts on the server.
+These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
+After setting the variable, either rerun `scripts/run-python-test.sh`
+or run offline and online tests with `tox` directly::
+
+    tox -e py
+
+Each test run creates new accounts.
+
+Developing the Bindings
+-----------------------
+
+If you want to develop or debug the bindings,
+you can create a testing development environment using `tox`::
+
+    export DCC_RS_DEV="$PWD"
+    export DCC_RS_TARGET=debug
+    tox -c python --devenv env -e py
+    . env/bin/activate
+
+Inside this environment the bindings are installed
+in editable mode (as if installed with `python -m pip install -e`)
+together with the testing dependencies like `pytest` and its plugins.
+
+You can then edit the source code in the development tree
+and quickly run `pytest` manually without waiting  for `tox`
+to recreating the virtual environment each time.

python/doc/changelog.rst

@@ -1,4 +0,0 @@
-Changelog for deltachat-core's Python bindings
-==============================================
-
-.. include:: ../CHANGELOG

python/doc/conf.py

@@ -1,138 +1,94 @@
-# -*- coding: utf-8 -*-
-#
-# devpi documentation build configuration file, created by
-# sphinx-quickstart on Mon Jun  3 16:11:22 2013.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import sys
-import os
+from pathlib import Path
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 from deltachat import __version__ as release
+
 version = ".".join(release.split(".")[:2])
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    #'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx.ext.viewcode',
-    'breathe',
-    #'sphinx.ext.githubpages',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.viewcode",
+    "breathe",
+    "sphinx_rtd_theme",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+project = "Delta Chat"
+copyright = "2023, Delta Chat contributors"
+author = "Delta Chat contributors"
 
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['sketch', '_build', "attic"]
+exclude_patterns = ["sketch", "_build", "attic", "Thumbs.db", ".DS_Store"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- breathe options ------
 
-breathe_projects = {
-    "deltachat": "../../docs/xml/"
-}
+breathe_projects = {"deltachat": Path("../../docs/xml/")}
 
 breathe_default_project = "deltachat"
 
 # -- Options for HTML output ---------------------------------------------------
 
-sys.path.append(os.path.abspath('_themes'))
-html_theme_path = ['_themes']
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-# html_theme = 'flask'
-html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-html_theme_options = {
-    'logo': '_static/delta-chat.svg',
-    'font_size': "1.1em",
-    'caption_font_size': "0.9em",
-    'code_font_size': "1.1em",
-
-
-}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = ["_themes"]
+html_theme = "sphinx_rtd_theme"
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
@@ -141,51 +97,34 @@ html_logo = "_static/delta-chat.svg"
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = '_static/favicon.ico'
+html_favicon = "_static/favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#
-html_sidebars = {
-    'index': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'searchbox.html'
-    ],
-    '**': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'relations.html',
-        'searchbox.html'
-    ]
-}
-
+# html_use_smartypants = True
 
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
@@ -194,71 +133,65 @@ html_show_sourcelink = False
 html_show_sphinx = False
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-html_use_opensearch = 'https://doc.devpi.net'
+html_use_opensearch = "https://doc.devpi.net"
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'deltachat-python'
+htmlhelp_basename = "deltachat-python"
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-'pointsize': '12pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '12pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'devpi.tex', u'deltachat documentation',
-   u'holger krekel', 'manual'),
+    ("index", "devpi.tex", "deltachat documentation", "holger krekel", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'deltachat', u'deltachat documentation',
-     [u'holger krekel'], 1)
-]
+man_pages = [("index", "deltachat", "deltachat documentation", ["holger krekel"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -267,30 +200,38 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'devpi', u'devpi Documentation',
-   u'holger krekel', 'devpi', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "devpi",
+        "devpi Documentation",
+        "holger krekel",
+        "devpi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {"http://docs.python.org/": None}
 
 # autodoc options
 autodoc_member_order = "bysource"
+
+
 # always document __init__ functions
 def skip(app, what, name, obj, skip, options):
     return skip
 
+
 def setup(app):
     app.connect("autodoc-skip-member", skip)
-

python/doc/index.rst

@@ -1,41 +1,44 @@
-deltachat python bindings
-=========================
+Delta Chat Python bindings, new and old
+=======
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
-
-- :doc:`api` is a high level interface to deltachat-core.
-
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
-
-
-
-getting started
----------------
+`Delta Chat <https://delta.chat/>`_ provides two kinds of Python bindings
+to the `Rust Core <https://github.com/deltachat/deltachat-core-rust>`_:
+JSON-RPC bindings and CFFI bindings.
+When starting a new project it is recommended to use JSON-RPC bindings,
+which are used in the Delta Chat Desktop app through generated Typescript-bindings. 
+The Python JSON-RPC bindings are maintained by Delta Chat core developers. 
+Most existing bot projects and many tests in Delta Chat's own core library
+still use the CFFI-bindings, and it is going to be maintained certainly also in 2024. 
+New APIs might however only appear in the JSON-RPC bindings, 
+as the CFFI bindings are increasingly in maintenance-only mode. 
 
 .. toctree::
    :maxdepth: 2
+   :caption: JSON-RPC Bindings
 
-   install
-   examples
+   jsonrpc/intro
+   jsonrpc/install
+   jsonrpc/examples
+   jsonrpc/reference
+   jsonrpc/develop
 
 .. toctree::
-   :hidden:
+   :maxdepth: 2
+   :caption: CFFI Bindings
 
-   links
-   changelog
-   api
-   lapi
-   plugins
-
-..
-    Indices and tables
-    ==================
-
-    * :ref:`genindex`
-    * :ref:`modindex`
-    * :ref:`search`
+   cffi/intro
+   cffi/install
+   cffi/examples
+   cffi/manylinux
+   cffi/tests
+   cffi/api
+   cffi/lapi
+   cffi/plugins
 
+.. _`deltachat`: https://delta.chat
+.. _`deltachat-core repo`: https://github.com/deltachat
+.. _pip: http://pypi.org/project/pip/
+.. _virtualenv: http://pypi.org/project/virtualenv/
+.. _merlinux: http://merlinux.eu
+.. _pypi: http://pypi.org/
+.. _`issue-tracker`: https://github.com/deltachat/deltachat-core-rust

python/doc/install.rst

@@ -1,2 +0,0 @@
-
-.. include:: ../README.rst

python/doc/jsonrpc/develop.rst

@@ -0,0 +1,68 @@
+===========
+Development
+===========
+
+To develop JSON-RPC bindings,
+clone the `deltachat-core-rust <https://github.com/deltachat/deltachat-core-rust/>`_ repository::
+
+   git clone https://github.com/deltachat/deltachat-core-rust.git
+
+Testing
+=======
+
+To run online tests, set ``CHATMAIL_DOMAIN``
+to a domain of the email server
+that can be used to create testing accounts::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+Then run ``scripts/run-rpc-test.sh``
+to build debug version of ``deltachat-rpc-server``
+and run ``deltachat-rpc-client`` tests
+in a separate virtual environment managed by `tox <https://tox.wiki/>`_.
+
+Development Environment
+=======================
+
+Creating a new virtual environment
+to run the tests each time
+as ``scripts/run-rpc-test.sh`` does is slow
+if you are changing the tests or the code
+and want to rerun the tests each time.
+
+If you are developing the tests,
+it is better to create a persistent virtual environment.
+You can do this by running ``scripts/make-rpc-testenv.sh``.
+This creates a virtual environment ``venv`` which you can then enter with::
+
+   . venv/bin/activate
+
+Then you can run the tests with
+
+::
+
+    pytest deltachat-rpc-client/tests/
+
+Refer to `pytest documentation <https://docs.pytest.org/>` for details.
+
+If make the changes to Delta Chat core
+or Python bindings, you can rebuild the environment by rerunning
+``scripts/make-rpc-testenv.sh``.
+It is ok to rebuild the activated environment this way,
+you do not need to deactivate or reactivate the environment each time.
+
+Using REPL
+==========
+
+Once you have a development environment,
+you can quickly test things in REPL::
+
+   $ python
+   >>> from deltachat_rpc_client import *
+   >>> rpc = Rpc()
+   >>> rpc.start()
+   >>> dc = DeltaChat(rpc)
+   >>> system_info = dc.get_system_info()
+   >>> system_info["level"]
+   'awesome'
+   >>> rpc.close()

python/doc/jsonrpc/examples.rst

@@ -0,0 +1,19 @@
+Examples
+========
+
+Echo bot
+--------
+.. include:: ../../../deltachat-rpc-client/examples/echobot_no_hooks.py
+    :literal:
+
+Echo bot with hooks
+-------------------
+.. include:: ../../../deltachat-rpc-client/examples/echobot.py
+    :literal:
+
+Advanced echo bot
+-----------------
+
+.. include:: ../../../deltachat-rpc-client/examples/echobot_advanced.py
+    :literal:
+

python/doc/jsonrpc/install.rst

@@ -0,0 +1,36 @@
+Install
+=======
+
+To use JSON-RPC bindings for Delta Chat core you will need
+a ``deltachat-rpc-server`` binary which provides Delta Chat core API over JSON-RPC
+and a ``deltachat-rpc-client`` Python package which is a JSON-RPC client that starts ``deltachat-rpc-server`` process and uses JSON-RPC API.
+
+`Create a virtual environment <https://docs.python.org/3/library/venv.html>`__ if you
+don’t have one already and activate it::
+
+   $ python -m venv venv
+   $ . venv/bin/activate
+
+Install ``deltachat-rpc-server``
+--------------------------------
+
+To get ``deltachat-rpc-server`` binary you have three options:
+
+1. Install ``deltachat-rpc-server`` from PyPI using ``pip install deltachat-rpc-server``.
+2. Build and install ``deltachat-rpc-server`` from source with ``cargo install --git https://github.com/deltachat/deltachat-core-rust/ deltachat-rpc-server``.
+3. Download prebuilt release from https://github.com/deltachat/deltachat-core-rust/releases and install it into ``PATH``.
+
+Check that ``deltachat-rpc-server`` is installed and can run::
+
+   $ deltachat-rpc-server --version
+   1.131.4
+
+Then install ``deltachat-rpc-client`` with ``pip install deltachat-rpc-client``.
+
+Install ``deltachat-rpc-client``
+--------------------------------
+
+To get ``deltachat-rpc-client`` Python library you can:
+
+1. Install ``deltachat-rpc-client`` from PyPI using ``pip install deltachat-rpc-client``.
+2. Install ``deltachat-rpc-client`` from source with ``pip install git+https://github.com/deltachat/deltachat-core-rust.git@main#subdirectory=deltachat-rpc-client``.

python/doc/jsonrpc/intro.rst

@@ -0,0 +1,8 @@
+Introduction
+============
+
+JSON-RPC bindings are available via the `deltachat-rpc-client <https://pypi.org/project/deltachat-rpc-client/>`_ Python package.
+This package provides only the Python bindings and requires ``deltachat-rpc-server`` binary to be installed.
+`deltachat-rpc-server <https://pypi.org/project/deltachat-rpc-server/>`_ package provides ``deltachat-rpc-server`` binary for Linux, Windows, macOS and Android.
+
+RPC client connects to standalone Delta Chat RPC server ``deltachat-rpc-server`` and provides Python interface to it.

python/doc/jsonrpc/reference.rst

@@ -0,0 +1,5 @@
+API Reference
+=============
+
+.. automodule:: deltachat_rpc_client
+   :members:

python/doc/links.rst

@@ -1,11 +0,0 @@
-
-links
-================================
-
-.. _`deltachat`: https://delta.chat
-.. _`deltachat-core repo`: https://github.com/deltachat
-.. _pip: http://pypi.org/project/pip/
-.. _virtualenv: http://pypi.org/project/virtualenv/
-.. _merlinux: http://merlinux.eu
-.. _pypi: http://pypi.org/
-.. _`issue-tracker`: https://github.com/deltachat/deltachat-core

python/doc/make.bat

@@ -1,190 +0,0 @@
-@ECHO OFF
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set BUILDDIR=_build
-set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
-set I18NSPHINXOPTS=%SPHINXOPTS% .
-if NOT "%PAPER%" == "" (
-	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
-	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
-)
-
-if "%1" == "" goto help
-
-if "%1" == "help" (
-	:help
-	echo.Please use `make ^<target^>` where ^<target^> is one of
-	echo.  html       to make standalone HTML files
-	echo.  dirhtml    to make HTML files named index.html in directories
-	echo.  singlehtml to make a single large HTML file
-	echo.  pickle     to make pickle files
-	echo.  json       to make JSON files
-	echo.  htmlhelp   to make HTML files and a HTML help project
-	echo.  qthelp     to make HTML files and a qthelp project
-	echo.  devhelp    to make HTML files and a Devhelp project
-	echo.  epub       to make an epub
-	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
-	echo.  text       to make text files
-	echo.  man        to make manual pages
-	echo.  texinfo    to make Texinfo files
-	echo.  gettext    to make PO message catalogs
-	echo.  changes    to make an overview over all changed/added/deprecated items
-	echo.  linkcheck  to check all external links for integrity
-	echo.  doctest    to run all doctests embedded in the documentation if enabled
-	goto end
-)
-
-if "%1" == "clean" (
-	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
-	del /q /s %BUILDDIR%\*
-	goto end
-)
-
-if "%1" == "html" (
-	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
-	goto end
-)
-
-if "%1" == "dirhtml" (
-	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
-	goto end
-)
-
-if "%1" == "singlehtml" (
-	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
-	goto end
-)
-
-if "%1" == "pickle" (
-	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the pickle files.
-	goto end
-)
-
-if "%1" == "json" (
-	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the JSON files.
-	goto end
-)
-
-if "%1" == "htmlhelp" (
-	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run HTML Help Workshop with the ^
-.hhp project file in %BUILDDIR%/htmlhelp.
-	goto end
-)
-
-if "%1" == "qthelp" (
-	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run "qcollectiongenerator" with the ^
-.qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\devpi.qhcp
-	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\devpi.ghc
-	goto end
-)
-
-if "%1" == "devhelp" (
-	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished.
-	goto end
-)
-
-if "%1" == "epub" (
-	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The epub file is in %BUILDDIR%/epub.
-	goto end
-)
-
-if "%1" == "latex" (
-	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
-	goto end
-)
-
-if "%1" == "text" (
-	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The text files are in %BUILDDIR%/text.
-	goto end
-)
-
-if "%1" == "man" (
-	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The manual pages are in %BUILDDIR%/man.
-	goto end
-)
-
-if "%1" == "texinfo" (
-	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
-	goto end
-)
-
-if "%1" == "gettext" (
-	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
-	goto end
-)
-
-if "%1" == "changes" (
-	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.The overview file is in %BUILDDIR%/changes.
-	goto end
-)
-
-if "%1" == "linkcheck" (
-	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Link check complete; look for any errors in the above output ^
-or in %BUILDDIR%/linkcheck/output.txt.
-	goto end
-)
-
-if "%1" == "doctest" (
-	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Testing of doctests in the sources finished, look at the ^
-results in %BUILDDIR%/doctest/output.txt.
-	goto end
-)
-
-:end