feat: dc_accounts_set_push_device_token and dc_get_push_state APIs

This is what updated dependencies require.

Also update Rust used to build manylinux wheels
from 1.72.0 to 1.76.0.

.github/workflows/ci.yml

@@ -24,9 +24,11 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.75.0
+      RUSTUP_TOOLCHAIN: 1.76.0
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Install rustfmt and clippy
         run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
       - name: Cache rust cargo artifacts
@@ -42,7 +44,9 @@ jobs:
     name: cargo deny
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: EmbarkStudios/cargo-deny-action@v1
         with:
           arguments: --all-features --workspace
@@ -53,7 +57,9 @@ jobs:
     name: Check provider database
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Check provider database
         run: scripts/update-provider-database.sh
 
@@ -63,8 +69,9 @@ jobs:
     env:
       RUSTDOCFLAGS: -Dwarnings
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
       - name: Rustdoc
@@ -76,18 +83,20 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.75.0
+            rust: 1.76.0
           - os: windows-latest
-            rust: 1.75.0
+            rust: 1.76.0
           - os: macos-latest
-            rust: 1.75.0
+            rust: 1.76.0
 
-          # Minimum Supported Rust Version = 1.70.0
+          # Minimum Supported Rust Version = 1.74.0
           - os: ubuntu-latest
-            rust: 1.70.0
+            rust: 1.74.0
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install Rust ${{ matrix.rust }}
         run: rustup toolchain install --profile minimal ${{ matrix.rust }}
@@ -111,7 +120,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -120,7 +131,7 @@ jobs:
         run: cargo build -p deltachat_ffi --features jsonrpc
 
       - name: Upload C library
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug/libdeltachat.a
@@ -133,7 +144,9 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -142,7 +155,7 @@ jobs:
         run: cargo build -p deltachat-rpc-server
 
       - name: Upload deltachat-rpc-server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
@@ -152,8 +165,9 @@ jobs:
     name: Python lint
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install tox
         run: pip install tox
@@ -193,16 +207,18 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Download libdeltachat.a
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -243,10 +259,12 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -254,7 +272,7 @@ jobs:
         run: pip install tox
 
       - name: Download deltachat-rpc-server
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: target/debug

.github/workflows/deltachat-rpc-server.yml

@@ -21,22 +21,27 @@ jobs:
   # Build a version statically linked against musl libc
   # to avoid problems with glibc version incompatibility.
   build_linux:
-    name: Cross-compile deltachat-rpc-server for x86_64, i686, aarch64 and armv7 Linux
-    runs-on: ubuntu-22.04
+    name: Build deltachat-rpc-server for Linux
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [aarch64, armv7l, armv6l, i686, x86_64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install ziglang
-        run: pip install wheel ziglang==0.11.0
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
-        run: sh scripts/zig-rpc-server.sh
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-linux
 
-      - name: Upload dist directory with Linux binaries
-        uses: actions/upload-artifact@v3
+      - name: Upload binary
+        uses: actions/upload-artifact@v4
         with:
-          name: linux
-          path: dist/
+          name: deltachat-rpc-server-${{ matrix.arch }}-linux
+          path: result/bin/deltachat-rpc-server
           if-no-files-found: error
 
   build_windows:
@@ -44,32 +49,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: windows-latest
-            artifact: win32.exe
-            path: deltachat-rpc-server.exe
-            target: i686-pc-windows-msvc
-
-          - os: windows-latest
-            artifact: win64.exe
-            path: deltachat-rpc-server.exe
-            target: x86_64-pc-windows-msvc
-
-    runs-on: ${{ matrix.os }}
+        arch: [win32, win64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
-      - name: Setup rust target
-        run: rustup target add ${{ matrix.target }}
-
-      - name: Build
-        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
+      - name: Build deltachat-rpc-server binaries
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: deltachat-rpc-server-${{ matrix.artifact }}
-          path: target/${{ matrix.target}}/release/${{ matrix.path }}
+          name: deltachat-rpc-server-${{ matrix.arch }}
+          path: result/bin/deltachat-rpc-server.exe
           if-no-files-found: error
 
   build_macos:
@@ -77,13 +73,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch: x86_64
-          - arch: aarch64
+        arch: [x86_64, aarch64]
 
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Setup rust target
         run: rustup target add ${{ matrix.arch }}-apple-darwin
@@ -92,7 +88,7 @@ jobs:
         run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.arch }}-apple-darwin --features vendored
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-macos
           path: target/${{ matrix.arch }}-apple-darwin/release/deltachat-rpc-server
@@ -100,53 +96,85 @@ jobs:
 
   publish:
     name: Build wheels and upload binaries to the release
-    needs: ["build_linux", "build_windows", "build_macos"]
+    needs: ["build_linux", "build_macos"]
     permissions:
       contents: write
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Download Linux binaries
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
         with:
-          name: linux
-          path: dist/
+          show-progress: false
 
-      - name: Download win32 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux aarch64 binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win32.exe
-          path: deltachat-rpc-server-win32.exe.d
+          name: deltachat-rpc-server-aarch64-linux
+          path: deltachat-rpc-server-aarch64-linux.d
 
-      - name: Download win64 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux armv7l binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win64.exe
-          path: deltachat-rpc-server-win64.exe.d
+          name: deltachat-rpc-server-armv7l-linux
+          path: deltachat-rpc-server-armv7l-linux.d
+
+      - name: Download Linux armv6l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv6l-linux
+          path: deltachat-rpc-server-armv6l-linux.d
+
+      - name: Download Linux i686 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-i686-linux
+          path: deltachat-rpc-server-i686-linux.d
+
+      - name: Download Linux x86_64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-linux
+          path: deltachat-rpc-server-x86_64-linux.d
+
+      - name: Download Win32 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win32
+          path: deltachat-rpc-server-win32.d
+
+      - name: Download Win64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win64
+          path: deltachat-rpc-server-win64.d
 
       - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-x86_64-macos
           path: deltachat-rpc-server-x86_64-macos.d
 
       - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-aarch64-macos
           path: deltachat-rpc-server-aarch64-macos.d
 
       - name: Flatten dist/ directory
         run: |
-          mv deltachat-rpc-server-win32.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
-          mv deltachat-rpc-server-win64.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
+          mkdir -p dist
+          mv deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-linux
+          mv deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-armv7l-linux
+          mv deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-armv6l-linux
+          mv deltachat-rpc-server-i686-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-i686-linux
+          mv deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-linux
+          mv deltachat-rpc-server-win32.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
+          mv deltachat-rpc-server-win64.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
           mv deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-macos
           mv deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-macos
 
       # Python 3.11 is needed for tomllib used in scripts/wheel-rpc-server.py
       - name: Install python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
@@ -160,6 +188,7 @@ jobs:
         run: ls -l dist/
 
       - name: Upload binaries to the GitHub release
+        if: github.event_name == 'release'
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |

.github/workflows/jsonrpc-client-npm-package.yml

@@ -13,8 +13,9 @@ jobs:
     steps:
       - name: Install tree
         run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -49,7 +50,7 @@ jobs:
           ls -lah
           mv $(find deltachat-jsonrpc-client-*) $DELTACHAT_JSONRPC_TAR_GZ
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-jsonrpc-client.tgz
           path: deltachat-jsonrpc/typescript/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}

.github/workflows/jsonrpc.yml

@@ -14,7 +14,9 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:

.github/workflows/node-docs.yml

@@ -14,7 +14,9 @@ jobs:
   generate:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3

.github/workflows/node-package.yml

@@ -14,8 +14,9 @@ jobs:
       matrix:
         os: [macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -56,7 +57,7 @@ jobs:
           tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: node/${{ matrix.os }}.tar.gz
@@ -74,8 +75,9 @@ jobs:
       - name: Change working directory owner
         run: chown root:root .
 
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -125,7 +127,7 @@ jobs:
           tar -zcvf "linux.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux
           path: node/linux.tar.gz
@@ -137,8 +139,9 @@ jobs:
     steps:
       - name: Install tree
         run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v2
         with:
           node-version: "18"
@@ -165,23 +168,23 @@ jobs:
           node --version
           echo $DELTACHAT_NODE_TAR_GZ
       - name: Download Linux prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: linux
       - name: Download macOS prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: macos-latest
       - name: Download Windows prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: windows-latest
       - shell: bash
         run: |
           mkdir node/prebuilds
-          tar -xvzf linux/linux.tar.gz -C node/prebuilds
-          tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
-          tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
+          tar -xvzf linux.tar.gz -C node/prebuilds
+          tar -xvzf macos-latest.tar.gz -C node/prebuilds
+          tar -xvzf windows-latest.tar.gz -C node/prebuilds
           tree node/prebuilds
           rm -rf linux macos-latest windows-latest
       - name: Install dependencies without running scripts
@@ -201,7 +204,7 @@ jobs:
           ls -lah
           mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
       - name: Upload prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-node.tgz
           path: ${{ env.DELTACHAT_NODE_TAR_GZ }}

.github/workflows/node-tests.yml

@@ -23,8 +23,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"

.github/workflows/repl.yml

@@ -10,15 +10,17 @@ on:
 jobs:
   build_repl:
     name: Build REPL example
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build
-        run: cargo build -p deltachat-repl --features vendored
-
+        run: nix build .#deltachat-repl-win64
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: repl.exe
-          path: "target/debug/deltachat-repl.exe"
+          path: "result/bin/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -1,4 +1,4 @@
-name: Build & Deploy Documentation on rs.delta.chat
+name: Build & Deploy Documentation on rs.delta.chat, c.delta.chat, py.delta.chat
 
 on:
   push:
@@ -6,11 +6,13 @@ on:
       - main
 
 jobs:
-  build:
+  build-rs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat --no-deps --document-private-items
@@ -22,3 +24,41 @@ jobs:
           HOST: "delta.chat"
           SOURCE: "target/doc"
           TARGET: "/var/www/html/rs/"
+
+  build-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build Python documentation
+        run: nix build .#python-docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+
+  build-c:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build C documentation
+        run: nix build .#docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"

.github/workflows/upload-ffi-docs.yml

@@ -14,15 +14,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat_ffi --no-deps
       - name: Upload to cffi.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/cffi/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

.github/workflows/upload-python-docs.yml

@@ -1,25 +0,0 @@
-name: Build & Deploy Documentation on py.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Build Python documentation
-        run: scripts/build-python-docs.sh
-      - name: Upload to py.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: delta
-          KEY: ${{ secrets.CODESPEAK_KEY }}
-          HOST: "lists.codespeak.net"
-          SOURCE: "dist/html/"
-          TARGET: "/home/delta/build/master"

CHANGELOG.md

@@ -1,6 +1,81 @@
 # Changelog
 
-## [1.135.0] - 2024-02-14
+## [1.135.1] - 2024-02-20
+
+### Features / Changes
+
+- Sync self-avatar across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Sync Config::Selfstatus across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Remove webxdc sending limit.
+
+### Fixes
+
+- Never encrypt `{vc,vg}-request` SecureJoin messages.
+- Apply Autocrypt headers if timestamp is unchanged.
+- `Context::get_info`: Report displayname as "displayname" (w/o underscore).
+
+### Tests
+
+- Mock `SystemTime::now()` for the tests.
+- Add a test on protection message sort timestamp ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+### Build system
+
+- Add flake.nix.
+- Add footer template for git-cliff.
+
+### CI
+
+- Update GitHub Actions `actions/upload-artifact`, `actions/download-artifact`, `actions/checkout`.
+- Build deltachat-repl for Windows with nix.
+- Build deltachat-rpc-server with nix.
+- Try to upload deltachat-rpc-server only on release.
+
+### Refactor
+
+- `create_keypair`: Remove unnecessary `map_err`.
+- Return error with a cause when failing to export keys.
+- Rename incorrectly named variables in `create_keypair`.
+
+## [1.135.1] - 2024-02-20
+
+### CI
+
+- Update actions/upload-artifact.
+- Use actions/download-artifact@v4.
+- Replace download-artifact v1 with v4.
+- Update to actions/checkout@v4.
+- Fixup node-package.yml after artifact actions upgrade.
+
+### Features / Changes
+
+- Mock SystemTime::now() for the tests.
+- Remove webxdc sending limit.
+- Sync self-avatar across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Sync Config::Selfstatus across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+
+### Fixes
+
+- Context::get_info: Report displayname as "displayname" (w/o underscore).
+- Never encrypt {vc,vg}-request.
+
+### Other
+
+- Cleanup changelog ([#5265](https://github.com/deltachat/deltachat-core-rust/pull/5265))
+
+somehow a whole issue sneaked in :).
+
+### Refactor
+
+- create_keypair: Remove unnecessary map_err.
+- Return error with a cause when failing to export keys.
+- Rename incorrectly named variables in create_keypair.
+
+### Tests
+
+- Add a test on protection message sort timestamp ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+## [1.135.0] - 2024-02-13
 
 ### Features / Changes
 
@@ -9,6 +84,7 @@
 - Context::set_config(): Restart IO scheduler if needed ([#5111](https://github.com/deltachat/deltachat-core-rust/pull/5111)).
 - Server_sent_unsolicited_exists(): Log folder name.
 - Cache system time instead of looking at the clock several times in a row.
+- Basic self-reporting ([#5129](https://github.com/deltachat/deltachat-core-rust/pull/5129)).
 
 ### Fixes
 
@@ -49,47 +125,11 @@
 ### Other
 
 - Update welcome image, thanks @paulaluap
-.
 - Merge pull request #5243 from deltachat/dependabot/cargo/pin-project-1.1.4
-
-.
 - Merge pull request #5241 from deltachat/dependabot/cargo/futures-lite-2.2.0
-
-.
 - Merge pull request #5236 from deltachat/dependabot/cargo/chrono-0.4.33
-
-.
 - Merge pull request #5235 from deltachat/dependabot/cargo/image-0.24.8
 
-.
-- Basic self-reporting, core part ([#5129](https://github.com/deltachat/deltachat-core-rust/pull/5129))
-
-Part of https://github.com/deltachat/deltachat-android/issues/2909
-
-For now, this is only sending a few basic metrics..
-- Do not change db schema in an incompatible way ([#5254](https://github.com/deltachat/deltachat-core-rust/pull/5254))
-
-PR #5099 removed some columns in the database that were actually in use.
-
-usually, to not worsen UX unnecessarily
-(releases take time - in between, "Add Second Device", "Backup" etc.
-would fail), we try to avoid such schema changes (checking for
-db-version would avoid import etc. but would still worse UX),
-see discussion at #2294.
-
-these are the errors, the user will be confronted with otherwise:
-
-<img width=400
-src=https://github.com/deltachat/deltachat-core-rust/assets/9800740/e3f0fd6e-a7a9-43f6-9023-0ae003985425>
-
-it is not great to maintain the old columns, but well :)
-
-as no official releases with newer cores are rolled out yet, i think, it
-is fine to change the "107" migration
-and not copy things a second time in a newer migration.
-
-(this issue happens to me during testing, and is probably also the issue
-reported by @lk108 for ubuntu-touch).
 
 ### Refactor
 
@@ -3576,3 +3616,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.133.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.1...v1.133.2
 [1.134.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.2...v1.134.0
 [1.135.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.134.0...v1.135.0
+[1.135.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.135.0...v1.135.1

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.135.0"
+version = "1.135.1"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.70"
+rust-version = "1.74"
 
 [profile.dev]
 debug = 0
@@ -32,6 +32,7 @@ strip = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+deltachat-time = { path = "./deltachat-time" }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
@@ -55,7 +56,7 @@ futures-lite = "2.2.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.8", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.24.9", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
@@ -68,7 +69,7 @@ num-traits = "0.2"
 once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.10", default-features = false }
+pgp = { version = "0.11", default-features = false }
 pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
@@ -77,7 +78,7 @@ quoted_printable = "0.5"
 rand = "0.8"
 regex = "1.10"
 reqwest = { version = "0.11.24", features = ["json"] }
-rusqlite = { version = "0.30", features = ["sqlcipher"] }
+rusqlite = { version = "0.31", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1"
@@ -88,7 +89,7 @@ smallvec = "1"
 strum = "0.26"
 strum_macros = "0.26"
 tagger = "4.3.4"
-textwrap = "0.16.0"
+textwrap = "0.16.1"
 thiserror = "1"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
@@ -99,14 +100,6 @@ toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 
-# Pin OpenSSL to 3.1 releases.
-# OpenSSL 3.2 has a regression tracked at <https://github.com/openssl/openssl/issues/23376>
-# which results in broken `deltachat-rpc-server` binaries when cross-compiled using Zig toolchain.
-# See <https://github.com/deltachat/deltachat-core-rust/issues/5206> for Delta Chat issue.
-# According to <https://www.openssl.org/policies/releasestrat.html>
-# 3.1 branch will be supported until 2025-03-14.
-openssl-src = "~300.1"
-
 [dev-dependencies]
 ansi_term = "0.12.0"
 anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
@@ -128,6 +121,7 @@ members = [
   "deltachat-rpc-server",
   "deltachat-ratelimit",
   "deltachat-repl",
+  "deltachat-time",
   "format-flowed",
 ]
 

cliff.toml

@@ -77,3 +77,17 @@ body = """
 """
 # remove the leading and trailing whitespace from the template
 trim = true
+footer = """
+{% for release in releases -%}
+    {% if release.version -%}
+        {% if release.previous.version -%}
+            [{{ release.version | trim_start_matches(pat="v") }}]: \
+                https://github.com/deltachat/deltachat-core-rust\
+                    /compare/{{ release.previous.version }}..{{ release.version }}
+        {% endif -%}
+    {% else -%}
+        [unreleased]: https://github.com/deltachat/deltachat-core-rust\
+            /compare/{{ release.previous.version }}..HEAD
+    {% endif -%}
+{% endfor %}
+"""

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.135.0"
+version = "1.135.1"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -686,6 +686,24 @@ int             dc_get_connectivity          (dc_context_t* context);
 char*           dc_get_connectivity_html     (dc_context_t* context);
 
 
+#define DC_PUSH_NOT_CONNECTED 0
+#define DC_PUSH_HEARTBEAT     1
+#define DC_PUSH_CONNECTED     2
+
+/**
+ * Get the current push notification state.
+ * One of:
+ * - DC_PUSH_NOT_CONNECTED
+ * - DC_PUSH_HEARTBEAT
+ * - DC_PUSH_CONNECTED
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return Push notification state.
+ */
+int              dc_get_push_state           (dc_context_t* context);
+
+
 /**
  * Standalone version of dc_accounts_all_work_done().
  * Only used by the python tests.
@@ -3165,6 +3183,16 @@ void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
  */
 int            dc_accounts_background_fetch    (dc_accounts_t* accounts, uint64_t timeout);
 
+
+/**
+ * Sets device token for Apple Push Notification service.
+ * Returns immediately.
+ *
+ * @memberof dc_accounts_t
+ * @param token Hexadecimal device token
+ */
+void           dc_accounts_set_push_device_token (dc_accounts_t* accounts, const char *token);
+
 /**
  * Create the event emitter that is used to receive events.
  *

deltachat-ffi/src/lib.rs

@@ -26,7 +26,7 @@ use anyhow::Context as _;
 use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, ContactId, Origin};
-use deltachat::context::Context;
+use deltachat::context::{Context, ContextBuilder};
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
@@ -34,7 +34,6 @@ use deltachat::message::MsgId;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
-use deltachat::stock_str::StockStrings;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
@@ -104,12 +103,11 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            as_path(dbfile),
-            id,
-            Events::new(),
-            StockStrings::new(),
-        ))
+        block_on(
+            ContextBuilder::new(as_path(dbfile).to_path_buf())
+                .with_id(id)
+                .open(),
+        )
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -133,12 +131,11 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     }
 
     let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile),
-        id,
-        Events::new(),
-        StockStrings::new(),
-    )) {
+    match block_on(
+        ContextBuilder::new(as_path(dbfile).to_path_buf())
+            .with_id(id)
+            .build(),
+    ) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
             eprintln!("failed to create context: {err:#}");
@@ -387,7 +384,7 @@ pub unsafe extern "C" fn dc_get_connectivity(context: *const dc_context_t) -> li
         return 0;
     }
     let ctx = &*context;
-    block_on(async move { ctx.get_connectivity().await as u32 as libc::c_int })
+    block_on(ctx.get_connectivity()) as u32 as libc::c_int
 }
 
 #[no_mangle]
@@ -410,6 +407,16 @@ pub unsafe extern "C" fn dc_get_connectivity_html(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_push_state(context: *const dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_push_state()");
+        return 0;
+    }
+    let ctx = &*context;
+    block_on(ctx.push_state()) as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_all_work_done(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -4922,6 +4929,29 @@ pub unsafe extern "C" fn dc_accounts_background_fetch(
     1
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_set_push_device_token(
+    accounts: *mut dc_accounts_t,
+    token: *const libc::c_char,
+) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_set_push_device_token()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    let token = to_string_lossy(token);
+
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        if let Err(err) = accounts.set_push_device_token(&token).await {
+            accounts.emit_event(EventType::Error(format!(
+                "Failed to set notify token: {err:#}."
+            )));
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     accounts: *mut dc_accounts_t,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.135.0"
+version = "1.135.1"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -17,7 +17,7 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.9.0"
+tempfile = "3.10.1"
 log = "0.4"
 async-channel = { version = "2.0.0" }
 futures = { version = "0.3.30" }
@@ -26,7 +26,7 @@ yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }
 sanitize-filename = "0.5"
-walkdir = "2.3.3"
+walkdir = "2.5.0"
 base64 = "0.21"
 
 # optional dependencies

deltachat-jsonrpc/typescript/package.json

@@ -53,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.135.0"
+  "version": "1.135.1"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.135.0"
+version = "1.135.1"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -9,9 +9,9 @@ ansi_term = "0.12.1"
 anyhow = "1"
 deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
-log = "0.4.20"
+log = "0.4.21"
 pretty_env_logger = "0.5"
-rusqlite = "0.30"
+rusqlite = "0.31"
 rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 

deltachat-repl/src/main.rs

@@ -9,7 +9,6 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
@@ -20,8 +19,7 @@ use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::stock_str::StockStrings;
-use deltachat::{EventType, Events};
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
@@ -312,7 +310,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]), 0, Events::new(), StockStrings::new()).await?;
+    let context = ContextBuilder::new(args[1].clone().into())
+        .with_id(1)
+        .open()
+        .await?;
 
     let events = context.get_event_emitter();
     tokio::task::spawn(async move {

deltachat-rpc-client/tests/test_securejoin.py

@@ -164,6 +164,21 @@ def test_qr_readreceipt(acfactory) -> None:
     assert not bob.get_chat_by_contact(bob_contact_charlie)
 
 
+def test_setup_contact_resetup(acfactory) -> None:
+    """Tests that setup contact works after Alice resets the device and changes the key."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    alice = acfactory.resetup_account(alice)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+
 def test_verified_group_recovery(acfactory) -> None:
     """Tests verified group recovery by reverifying a member and sending a message in a group."""
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.135.0"
+version = "1.135.1"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-time/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "deltachat-time"
+version = "1.0.0"
+description = "Time-related tools"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]

deltachat-time/src/lib.rs

@@ -0,0 +1,35 @@
+#![allow(missing_docs)]
+
+use std::sync::RwLock;
+use std::time::{Duration, SystemTime};
+
+static SYSTEM_TIME_SHIFT: RwLock<Duration> = RwLock::new(Duration::new(0, 0));
+
+/// Fake struct for mocking `SystemTime::now()` for test purposes. You still need to use
+/// `SystemTime` as a struct representing a system time.
+pub struct SystemTimeTools();
+
+impl SystemTimeTools {
+    pub const UNIX_EPOCH: SystemTime = SystemTime::UNIX_EPOCH;
+
+    pub fn now() -> SystemTime {
+        return SystemTime::now() + *SYSTEM_TIME_SHIFT.read().unwrap();
+    }
+
+    /// Simulates a system clock forward adjustment by `duration`.
+    pub fn shift(duration: Duration) {
+        *SYSTEM_TIME_SHIFT.write().unwrap() += duration;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        SystemTimeTools::shift(Duration::from_secs(60));
+        let t = SystemTimeTools::now();
+        assert!(t > SystemTime::now());
+    }
+}

deny.toml

@@ -1,5 +1,4 @@
 [advisories]
-unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
@@ -10,6 +9,12 @@ ignore = [
     # There is no fix at the time of writing this (2023-11-28).
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
+
+    # Unmaintained ansi_term
+    "RUSTSEC-2021-0139",
+
+    # Unmaintained encoding
+    "RUSTSEC-2021-0153",
 ]
 
 [bans]
@@ -56,6 +61,7 @@ skip = [
      { name = "spki", version = "0.6.0" },
      { name = "syn", version = "1.0.109" },
      { name = "time", version = "<0.3" },
+     { name = "toml_edit", version = "0.21.1" },
      { name = "untrusted", version = "0.7.1" },
      { name = "wasi", version = "<0.11" },
      { name = "windows_aarch64_gnullvm", version = "<0.52" },
@@ -68,6 +74,7 @@ skip = [
      { name = "windows_x86_64_gnullvm", version = "<0.52" },
      { name = "windows_x86_64_gnu", version = "<0.52" },
      { name = "windows_x86_64_msvc", version = "<0.52" },
+     { name = "winnow", version = "0.5.40" },
 ]
 
 

flake.lock

@@ -0,0 +1,163 @@
+{
+  "nodes": {
+    "fenix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1707891749,
+        "narHash": "sha256-SeikNYElHgv8uVMbiA9/pU3Cce7ssIsiM8CnEiwd1Nc=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "3115aab064ef38cccd792c45429af8df43d6d277",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1705309234,
+        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "naersk": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1698420672,
+        "narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "aeb58d5e8faead8980a807c840232697982d47b9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "naersk",
+        "type": "github"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1705332318,
+        "narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3449dc925982ad46246cfc36469baf66e1b64f17",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1707689078,
+        "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1707743206,
+        "narHash": "sha256-AehgH64b28yKobC/DAWYZWkJBxL/vP83vkY+ag2Hhy4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2d627a2a704708673e56346fcb13d25344b8eaf3",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1707689078,
+        "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "fenix": "fenix",
+        "flake-utils": "flake-utils",
+        "naersk": "naersk",
+        "nix-filter": "nix-filter",
+        "nixpkgs": "nixpkgs_3"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1707849817,
+        "narHash": "sha256-If6T0MDErp3/z7DBlpG4bV46IPP+7BWSlgTI88cmbw0=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "a02a219773629686bd8ff123ca1aa995fa50d976",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,333 @@
+{
+  description = "Delta Chat core";
+  inputs = {
+    fenix.url = "github:nix-community/fenix";
+    flake-utils.url = "github:numtide/flake-utils";
+    naersk.url = "github:nix-community/naersk";
+    nix-filter.url = "github:numtide/nix-filter";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+  };
+  outputs = { self, nixpkgs, flake-utils, nix-filter, naersk, fenix }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        fenixPkgs = fenix.packages.${system};
+        naersk' = pkgs.callPackage naersk { };
+        manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
+        cargoLock = {
+          lockFile = ./Cargo.lock;
+          outputHashes = {
+            "email-0.0.20" = "sha256-rV4Uzqt2Qdrfi5Ti1r+Si1c2iW1kKyWLwOgLkQ5JGGw=";
+            "encoded-words-0.2.0" = "sha256-KK9st0hLFh4dsrnLd6D8lC6pRFFs8W+WpZSGMGJcosk=";
+            "lettre-0.9.2" = "sha256-+hU1cFacyyeC9UGVBpS14BWlJjHy90i/3ynMkKAzclk=";
+          };
+        };
+        mkRustPackage = packageName:
+          naersk'.buildPackage {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+          };
+        pkgsWin64 = pkgs.pkgsCross.mingwW64;
+        mkWin64RustPackage = packageName:
+          let
+            rustTarget = "x86_64-pc-windows-gnu";
+          in
+          let
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              pkgsWin64.stdenv.cc
+              pkgsWin64.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+          };
+
+        pkgsWin32 = pkgs.pkgsCross.mingw32;
+        mkWin32RustPackage = packageName:
+          let
+            rustTarget = "i686-pc-windows-gnu";
+          in
+          let
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+
+            # Get rid of MCF Gthread library.
+            # See <https://github.com/NixOS/nixpkgs/issues/156343>
+            # and <https://discourse.nixos.org/t/statically-linked-mingw-binaries/38395>
+            # for details.
+            #
+            # Use DWARF-2 instead of SJLJ for exception handling.
+            winCC = pkgsWin32.buildPackages.wrapCC (
+              (pkgsWin32.buildPackages.gcc-unwrapped.override
+                ({
+                  threadsCross = {
+                    model = "win32";
+                    package = null;
+                  };
+                })).overrideAttrs (oldAttr: rec{
+                configureFlags = oldAttr.configureFlags ++ [
+                  "--disable-sjlj-exceptions --with-dwarf2"
+                ];
+              })
+            );
+            winStdenv = pkgsWin32.buildPackages.overrideCC pkgsWin32.stdenv winCC;
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              winCC
+              pkgsWin32.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            LD = "${winCC}/bin/${winCC.targetPrefix}cc";
+          };
+
+        mkCrossRustPackage = rustTarget: crossTarget: packageName:
+          let
+            pkgsCross = import nixpkgs {
+              system = system;
+              crossSystem.config = crossTarget;
+            };
+          in
+          let
+            toolchain = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naersk-lib = pkgs.callPackage naersk {
+              cargo = toolchain;
+              rustc = toolchain;
+            };
+          in
+          naersk-lib.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+
+            OPENSSL_LIB_DIR = "${pkgsCross.pkgsStatic.openssl.out}/lib";
+            OPENSSL_INCLUDE_DIR = "${pkgsCross.pkgsStatic.openssl.dev}/include";
+            OPENSSL_STATIC = "1";
+            OPENSSL_NO_VENDOR = "1";
+          };
+
+        mk-aarch64-RustPackage = mkCrossRustPackage "aarch64-unknown-linux-musl" "aarch64-unknown-linux-musl";
+        mk-i686-RustPackage = mkCrossRustPackage "i686-unknown-linux-musl" "i686-unknown-linux-musl";
+        mk-x86_64-RustPackage = mkCrossRustPackage "x86_64-unknown-linux-musl" "x86_64-unknown-linux-musl";
+        mk-armv7l-RustPackage = mkCrossRustPackage "armv7-unknown-linux-musleabihf" "armv7l-unknown-linux-musleabihf";
+        mk-armv6l-RustPackage = mkCrossRustPackage "arm-unknown-linux-musleabihf" "armv6l-unknown-linux-musleabihf";
+      in
+      {
+        formatter = pkgs.nixpkgs-fmt;
+
+        packages = rec {
+          # Run with `nix run .#deltachat-repl foo.db`.
+          deltachat-repl = mkRustPackage "deltachat-repl";
+          deltachat-rpc-server = mkRustPackage "deltachat-rpc-server";
+
+          deltachat-repl-win64 = mkWin64RustPackage "deltachat-repl";
+          deltachat-rpc-server-win64 = mkWin64RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-win32 = mkWin32RustPackage "deltachat-repl";
+          deltachat-rpc-server-win32 = mkWin32RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-aarch64-linux = mk-aarch64-RustPackage "deltachat-repl";
+          deltachat-rpc-server-aarch64-linux = mk-aarch64-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-i686-linux = mk-i686-RustPackage "deltachat-repl";
+          deltachat-rpc-server-i686-linux = mk-i686-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-x86_64-linux = mk-x86_64-RustPackage "deltachat-repl";
+          deltachat-rpc-server-x86_64-linux = mk-x86_64-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-armv7l-linux = mk-armv7l-RustPackage "deltachat-repl";
+          deltachat-rpc-server-armv7l-linux = mk-armv7l-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-armv6l-linux = mk-armv6l-RustPackage "deltachat-repl";
+          deltachat-rpc-server-armv6l-linux = mk-armv6l-RustPackage "deltachat-rpc-server";
+
+          # Run `nix build .#docs` to get C docs generated in `./result/`.
+          docs =
+            pkgs.stdenv.mkDerivation {
+              pname = "docs";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./.;
+              nativeBuildInputs = [ pkgs.doxygen ];
+              buildPhase = ''scripts/run-doxygen.sh'';
+              installPhase = ''mkdir -p $out; cp -av deltachat-ffi/html deltachat-ffi/xml $out'';
+            };
+
+          libdeltachat =
+            pkgs.stdenv.mkDerivation rec {
+              pname = "libdeltachat";
+              version = manifest.version;
+              src = nix-filter.lib {
+                root = ./.;
+
+                # Include only necessary files
+                # to avoid rebuilds e.g. when README.md or flake.nix changes.
+                include = [
+                  ./benches
+                  ./assets
+                  ./Cargo.lock
+                  ./Cargo.toml
+                  ./CMakeLists.txt
+                  ./CONTRIBUTING.md
+                  ./deltachat_derive
+                  ./deltachat-ffi
+                  ./deltachat-jsonrpc
+                  ./deltachat-ratelimit
+                  ./deltachat-repl
+                  ./deltachat-rpc-client
+                  ./deltachat-time
+                  ./deltachat-rpc-server
+                  ./format-flowed
+                  ./release-date.in
+                  ./src
+                ];
+                exclude = [
+                  (nix-filter.lib.matchExt "nix")
+                  "flake.lock"
+                ];
+              };
+              cargoDeps = pkgs.rustPlatform.importCargoLock cargoLock;
+
+              nativeBuildInputs = [
+                pkgs.perl # Needed to build vendored OpenSSL.
+                pkgs.cmake
+                pkgs.rustPlatform.cargoSetupHook
+                pkgs.cargo
+              ];
+
+              postInstall = ''
+                substituteInPlace $out/include/deltachat.h \
+                  --replace __FILE__ '"${placeholder "out"}/include/deltachat.h"'
+              '';
+            };
+
+          deltachat-rpc-client =
+            pkgs.python3Packages.buildPythonPackage rec {
+              pname = "deltachat-rpc-client";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./deltachat-rpc-client;
+              format = "pyproject";
+              propagatedBuildInputs = [
+                pkgs.python3Packages.setuptools
+                pkgs.python3Packages.setuptools_scm
+              ];
+            };
+
+          deltachat-python =
+            pkgs.python3Packages.buildPythonPackage rec {
+              pname = "deltachat-python";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./python;
+              format = "pyproject";
+              buildInputs = [
+                libdeltachat
+              ];
+              nativeBuildInputs = [
+                pkgs.pkg-config
+              ];
+              propagatedBuildInputs = [
+                pkgs.python3Packages.setuptools
+                pkgs.python3Packages.setuptools_scm
+                pkgs.python3Packages.pkgconfig
+                pkgs.python3Packages.cffi
+                pkgs.python3Packages.imap-tools
+                pkgs.python3Packages.pluggy
+                pkgs.python3Packages.requests
+              ];
+            };
+          python-docs =
+            pkgs.stdenv.mkDerivation {
+              pname = "docs";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./.;
+              buildInputs = [
+                deltachat-python
+                deltachat-rpc-client
+                pkgs.python3Packages.breathe
+                pkgs.python3Packages.sphinx_rtd_theme
+              ];
+              nativeBuildInputs = [ pkgs.sphinx ];
+              buildPhase = ''sphinx-build -b html -a python/doc/ dist/html'';
+              installPhase = ''mkdir -p $out; cp -av dist/html $out'';
+            };
+        };
+      }
+    );
+}

node/test/test.mjs

@@ -239,7 +239,7 @@ describe('Basic offline Tests', function () {
       'delete_device_after',
       'delete_server_after',
       'deltachat_core_version',
-      'display_name',
+      'displayname',
       'download_limit',
       'e2ee_enabled',
       'entered_account_settings',

package.json

@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.135.0"
+  "version": "1.135.1"
 }

python/src/deltachat/account.py

@@ -375,22 +375,6 @@ class Account:
         dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
         return (x for x in iter_array(dc_array, lambda x: Message.from_db(self, x)) if x is not None)
 
-    def _wait_next_message_ids(self) -> List[int]:
-        """Return IDs of all next messages from all chats."""
-        dc_array = ffi.gc(lib.dc_wait_next_msgs(self._dc_context), lib.dc_array_unref)
-        return [lib.dc_array_get_id(dc_array, i) for i in range(lib.dc_array_get_cnt(dc_array))]
-
-    def wait_next_incoming_message(self) -> Message:
-        """Waits until the next incoming message
-        with ID higher than given is received and returns it."""
-        while True:
-            message_ids = self._wait_next_message_ids()
-            for msg_id in message_ids:
-                message = Message.from_db(self, msg_id)
-                if message and not message.is_from_self() and not message.is_from_device():
-                    self.set_config("last_msg_id", str(msg_id))
-                    return message
-
     def create_chat(self, obj) -> Chat:
         """Create a 1:1 chat with Account, Contact or e-mail address."""
         return self.create_contact(obj).create_chat()

python/src/deltachat/events.py

@@ -182,6 +182,12 @@ class FFIEventTracker:
                 print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)
                 break
 
+    def wait_securejoin_joiner_progress(self, target):
+        while True:
+            event = self.get_matching("DC_EVENT_SECUREJOIN_JOINER_PROGRESS")
+            if event.data2 >= target:
+                break
+
     def wait_idle_inbox_ready(self):
         """Has to be called after start_io() to wait for fetch_existing_msgs to run
         so that new messages are not mistaken for old ones:

python/src/deltachat/testplugin.py

@@ -10,7 +10,6 @@ import time
 import weakref
 import random
 from queue import Queue
-from threading import Event
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
@@ -116,7 +115,7 @@ def pytest_configure(config):
     deltachat.register_global_plugin(la)
 
 
-def pytest_report_header(config, startdir):
+def pytest_report_header(config):
     info = get_core_info()
     summary = [
         "Deltachat core={} sqlite={} journal_mode={}".format(
@@ -592,23 +591,16 @@ class ACFactory:
         return ac1.create_chat(ac2)
 
     def get_protected_chat(self, ac1: Account, ac2: Account):
-        class SetupPlugin:
-            def __init__(self) -> None:
-                self.member_added = Event()
-
-            @account_hookimpl
-            def ac_member_added(self, chat: deltachat.Chat, contact, actor, message):
-                self.member_added.set()
-
-        setupplugin = SetupPlugin()
-        ac1.add_account_plugin(setupplugin)
         chat = ac1.create_group_chat("Protected Group", verified=True)
         qr = chat.get_join_qr()
         ac2.qr_join_chat(qr)
-        setupplugin.member_added.wait()
-        msg = ac2.wait_next_incoming_message()
+        ac2._evtracker.wait_securejoin_joiner_progress(1000)
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev.data2)
+        assert msg is not None
         assert msg.text == "Messages are guaranteed to be end-to-end encrypted from now on."
-        msg = ac2.wait_next_incoming_message()
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg is not None
         assert "Member Me " in msg.text and " added by " in msg.text
         return chat
 

python/tests/test_1_online.py

@@ -44,21 +44,21 @@ def test_configure_generate_key(acfactory, lp):
     lp.sec("ac1: send unencrypted message to ac2")
     chat.send_text("message1")
     lp.sec("ac2: waiting for message from ac1")
-    msg_in = ac2.wait_next_incoming_message()
+    msg_in = ac2._evtracker.wait_next_incoming_message()
     assert msg_in.text == "message1"
     assert not msg_in.is_encrypted()
 
     lp.sec("ac2: send encrypted message to ac1")
     msg_in.chat.send_text("message2")
     lp.sec("ac1: waiting for message from ac2")
-    msg2_in = ac1.wait_next_incoming_message()
+    msg2_in = ac1._evtracker.wait_next_incoming_message()
     assert msg2_in.text == "message2"
     assert msg2_in.is_encrypted()
 
     lp.sec("ac1: send encrypted message to ac2")
     msg2_in.chat.send_text("message3")
     lp.sec("ac2: waiting for message from ac1")
-    msg3_in = ac2.wait_next_incoming_message()
+    msg3_in = ac2._evtracker.wait_next_incoming_message()
     assert msg3_in.text == "message3"
     assert msg3_in.is_encrypted()
 
@@ -520,7 +520,7 @@ def test_forward_encrypted_to_unencrypted(acfactory, lp):
     lp.sec("ac1: send encrypted message to ac2")
     txt = "This should be encrypted"
     chat.send_text(txt)
-    msg = ac2.wait_next_incoming_message()
+    msg = ac2._evtracker.wait_next_incoming_message()
     assert msg.text == txt
     assert msg.is_encrypted()
 

python/tests/test_3_offline.py

@@ -52,8 +52,8 @@ class TestOfflineAccountBasic:
     def test_wrong_db(self, tmp_path):
         p = tmp_path / "hello.db"
         p.write_text("123")
-        account = Account(str(p))
-        assert not account.is_open()
+        with pytest.raises(ValueError):
+            _account = Account(str(p))
 
     def test_os_name(self, tmp_path):
         p = tmp_path / "hello.db"

release-date.in

@@ -1 +1 @@
-2024-02-13
+2024-02-20

scripts/README.md

@@ -35,12 +35,8 @@ and an own build machine.
 
 - `run_all.sh` builds Python wheels
 
-- `zig-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Zig toolchain statically linked against musl libc.
-
 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.
 
-- `build-python-docs.sh` builds Python documentation into `dist/html/`.
-
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 

scripts/build-python-docs.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export DCC_RS_TARGET=debug
-export DCC_RS_DEV="$PWD"
-cargo build -p deltachat_ffi --features jsonrpc
-
-python3 -m venv venv
-venv/bin/pip install ./python
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/pip install sphinx breathe sphinx_rtd_theme
-venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/concourse/docs_wheels.yml

@@ -15,55 +15,6 @@ resources:
       tag_filter: "v*"
 
 jobs:
-  - name: doxygen
-    plan:
-      - get: deltachat-core-rust
-        trigger: true
-
-      # Build Doxygen documentation
-      - task: build-doxygen
-        config:
-          inputs:
-            - name: deltachat-core-rust
-          outputs:
-            - name: c-docs
-          image_resource:
-            source:
-              repository: alpine
-            type: registry-image
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache doxygen git
-                cd deltachat-core-rust
-                scripts/run-doxygen.sh
-                cd ..
-                cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
-
-      - task: upload-c-docs
-        config:
-          inputs:
-            - name: c-docs
-          image_resource:
-            type: registry-image
-            source:
-              repository: alpine
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache rsync openssh-client
-                mkdir -p ~/.ssh
-                chmod 700 ~/.ssh
-                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-                chmod 600 ~/.ssh/id_ed25519
-                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
   - name: python-x86_64
     plan:
       - get: deltachat-core-rust

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.72.0
+RUST_VERSION=1.76.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/wheel-rpc-server.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-"""Build Python wheels for deltachat-rpc-server.
-Run scripts/zig-rpc-server.sh first."""
+"""Build Python wheels for deltachat-rpc-server."""
 from pathlib import Path
 from wheel.wheelfile import WheelFile
 import tomllib
@@ -157,8 +156,13 @@ def main():
     )
     build_wheel(
         version,
-        "dist/deltachat-rpc-server-armv7-linux",
-        "py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
+        "dist/deltachat-rpc-server-armv7l-linux",
+        "py3-none-linux_armv7l.manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
+    )
+    build_wheel(
+        version,
+        "dist/deltachat-rpc-server-armv6l-linux",
+        "py3-none-linux_armv6l",
     )
     build_wheel(
         version,

scripts/zig-cc

@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-# /// pyproject
-# [run]
-# dependencies = [
-#   "ziglang==0.11.0"
-# ]
-# ///
-import os
-import subprocess
-import sys
-
-
-def flag_filter(flag: str) -> bool:
-    # Workaround for <https://github.com/sfackler/rust-openssl/issues/2043>.
-    if flag == "-latomic":
-        return False
-
-    if flag == "-lc":
-        return False
-    if flag == "-Wl,-melf_i386":
-        return False
-    if "self-contained" in flag and "crt" in flag:
-        return False
-    return True
-
-
-def main():
-    args = [flag for flag in sys.argv[1:] if flag_filter(flag)]
-    zig_target = os.environ["ZIG_TARGET"]
-    zig_cpu = os.environ.get("ZIG_CPU")
-    if zig_cpu:
-        zig_cpu_args = ["-mcpu=" + zig_cpu]
-        args = [x for x in args if not x.startswith("-march")]
-    else:
-        zig_cpu_args = []
-
-    # Disable atomics and use locks instead in OpenSSL.
-    # Zig toolchains do not provide atomics.
-    # This is a workaround for <https://github.com/deltachat/deltachat-core-rust/issues/4799>
-    args += ["-DBROKEN_CLANG_ATOMICS"]
-
-    subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "ziglang",
-            "cc",
-            "-target",
-            zig_target,
-            *zig_cpu_args,
-            *args,
-        ],
-        check=True,
-    )
-
-
-main()

scripts/zig-rpc-server.sh

@@ -1,50 +0,0 @@
-#!/bin/sh
-#
-# Build statically linked deltachat-rpc-server using zig.
-
-set -x
-set -e
-
-unset RUSTFLAGS
-
-# Pin Rust version to avoid uncontrolled changes in the compiler and linker flags.
-export RUSTUP_TOOLCHAIN=1.72.0
-
-rustup target add i686-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_I686_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="x86-linux-musl" \
-cargo build --release --target i686-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-rustup target add armv7-unknown-linux-musleabihf
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="arm-linux-musleabihf" \
-ZIG_CPU="generic+v7a+vfp3-d32+thumb2-neon" \
-cargo build --release --target armv7-unknown-linux-musleabihf -p deltachat-rpc-server --features vendored
-
-rustup target add x86_64-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="x86_64-linux-musl" \
-cargo build --release --target x86_64-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-rustup target add aarch64-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="aarch64-linux-musl" \
-cargo build --release --target aarch64-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-mkdir -p dist
-cp target/x86_64-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-linux
-cp target/i686-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-i686-linux
-cp target/aarch64-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-linux
-cp target/armv7-unknown-linux-musleabihf/release/deltachat-rpc-server dist/deltachat-rpc-server-armv7-linux

src/accounts.rs

@@ -17,8 +17,9 @@ use tokio::sync::oneshot;
 #[cfg(not(target_os = "ios"))]
 use tokio::time::{sleep, Duration};
 
-use crate::context::Context;
+use crate::context::{Context, ContextBuilder};
 use crate::events::{Event, EventEmitter, EventType, Events};
+use crate::push::PushSubscriber;
 use crate::stock_str::StockStrings;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -37,6 +38,9 @@ pub struct Accounts {
     /// This way changing a translation for one context automatically
     /// changes it for all other contexts.
     pub(crate) stockstrings: StockStrings,
+
+    /// Push notification subscriber shared between accounts.
+    push_subscriber: PushSubscriber,
 }
 
 impl Accounts {
@@ -73,8 +77,9 @@ impl Accounts {
             .context("failed to load accounts config")?;
         let events = Events::new();
         let stockstrings = StockStrings::new();
+        let push_subscriber = PushSubscriber::new();
         let accounts = config
-            .load_accounts(&events, &stockstrings, &dir)
+            .load_accounts(&events, &stockstrings, push_subscriber.clone(), &dir)
             .await
             .context("failed to load accounts")?;
 
@@ -84,6 +89,7 @@ impl Accounts {
             accounts,
             events,
             stockstrings,
+            push_subscriber,
         })
     }
 
@@ -120,13 +126,17 @@ impl Accounts {
         let account_config = self.config.new_account().await?;
         let dbfile = account_config.dbfile(&self.dir);
 
-        let ctx = Context::new(
-            &dbfile,
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .with_push_subscriber(self.push_subscriber.clone())
+            .build()
+            .await?;
+        // Try to open without a passphrase,
+        // but do not return an error if account is passphare-protected.
+        ctx.open("".to_string()).await?;
+
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -135,14 +145,15 @@ impl Accounts {
     /// Adds a new closed account.
     pub async fn add_closed_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account().await?;
+        let dbfile = account_config.dbfile(&self.dir);
 
-        let ctx = Context::new_closed(
-            &account_config.dbfile(&self.dir),
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .with_push_subscriber(self.push_subscriber.clone())
+            .build()
+            .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -337,6 +348,12 @@ impl Accounts {
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
+
+    /// Sets notification token for Apple Push Notification service.
+    pub async fn set_push_device_token(&mut self, token: &str) -> Result<()> {
+        self.push_subscriber.set_device_token(token).await;
+        Ok(())
+    }
 }
 
 /// Configuration file name.
@@ -522,24 +539,24 @@ impl Config {
         &self,
         events: &Events,
         stockstrings: &StockStrings,
+        push_subscriber: PushSubscriber,
         dir: &Path,
     ) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
 
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(
-                &account_config.dbfile(dir),
-                account_config.id,
-                events.clone(),
-                stockstrings.clone(),
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "failed to create context from file {:?}",
-                    account_config.dbfile(dir)
-                )
-            })?;
+            let dbfile = account_config.dbfile(dir);
+            let ctx = ContextBuilder::new(dbfile.clone())
+                .with_id(account_config.id)
+                .with_events(events.clone())
+                .with_stock_strings(stockstrings.clone())
+                .with_push_subscriber(push_subscriber.clone())
+                .build()
+                .await
+                .with_context(|| format!("failed to create context from file {:?}", &dbfile))?;
+            // Try to open without a passphrase,
+            // but do not return an error if account is passphare-protected.
+            ctx.open("".to_string()).await?;
 
             accounts.insert(account_config.id, ctx);
         }

src/blob.rs

@@ -8,6 +8,7 @@ use std::iter::FusedIterator;
 use std::path::{Path, PathBuf};
 
 use anyhow::{format_err, Context as _, Result};
+use base64::Engine as _;
 use futures::StreamExt;
 use image::{DynamicImage, GenericImageView, ImageFormat, ImageOutputFormat};
 use num_traits::FromPrimitive;
@@ -312,6 +313,30 @@ impl<'a> BlobObject<'a> {
         true
     }
 
+    /// Returns path to the stored Base64-decoded blob.
+    ///
+    /// If `data` represents an image of known format, this adds the corresponding extension to
+    /// `suggested_file_stem`.
+    pub(crate) async fn store_from_base64(
+        context: &Context,
+        data: &str,
+        suggested_file_stem: &str,
+    ) -> Result<String> {
+        let buf = base64::engine::general_purpose::STANDARD.decode(data)?;
+        let ext = if let Ok(format) = image::guess_format(&buf) {
+            if let Some(ext) = format.extensions_str().first() {
+                format!(".{ext}")
+            } else {
+                String::new()
+            }
+        } else {
+            String::new()
+        };
+        let blob =
+            BlobObject::create(context, &format!("{suggested_file_stem}{ext}"), &buf).await?;
+        Ok(blob.as_name().to_string())
+    }
+
     pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
 

src/chat.rs

@@ -6,7 +6,7 @@ use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{anyhow, bail, ensure, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
@@ -44,7 +44,7 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     buf_compress, create_id, create_outgoing_rfc724_mid, create_smeared_timestamp,
     create_smeared_timestamps, get_abs_path, gm2local_offset, improve_single_line_input,
-    smeared_time, strip_rtlo_characters, time, IsNoneOrEmpty,
+    smeared_time, strip_rtlo_characters, time, IsNoneOrEmpty, SystemTime,
 };
 use crate::webxdc::WEBXDC_SUFFIX;
 
@@ -3656,7 +3656,7 @@ pub enum MuteDuration {
     Forever,
 
     /// Chat is muted for a limited period of time.
-    Until(SystemTime),
+    Until(std::time::SystemTime),
 }
 
 impl rusqlite::types::ToSql for MuteDuration {

src/config.rs

@@ -5,9 +5,11 @@ use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
+use base64::Engine as _;
 use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
+use tokio::fs;
 
 use crate::blob::BlobObject;
 use crate::constants::{self, DC_VERSION_STR};
@@ -361,8 +363,8 @@ impl Config {
     ///
     /// This must be checked on both sides so that if there are different client versions, the
     /// synchronisation of a particular option is either done or not done in both directions.
-    /// Moreover, receivers of a config value need to check if a key can be synced because some
-    /// settings (e.g. Avatar path) could otherwise lead to exfiltration of files from a receiver's
+    /// Moreover, receivers of a config value need to check if a key can be synced because if it is
+    /// a file path, it could otherwise lead to exfiltration of files from a receiver's
     /// device if we assume an attacker to have control of a device in a multi-device setting or if
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
@@ -373,7 +375,11 @@ impl Config {
         }
         matches!(
             self,
-            Self::Displayname | Self::MdnsEnabled | Self::ShowEmails
+            Self::Displayname
+                | Self::MdnsEnabled
+                | Self::ShowEmails
+                | Self::Selfavatar
+                | Self::Selfstatus,
         )
     }
 
@@ -503,6 +509,23 @@ impl Context {
         }
     }
 
+    /// Executes [`SyncData::Config`] item sent by other device.
+    pub(crate) async fn sync_config(&self, key: &Config, value: &str) -> Result<()> {
+        let config_value;
+        let value = match key {
+            Config::Selfavatar if value.is_empty() => None,
+            Config::Selfavatar => {
+                config_value = BlobObject::store_from_base64(self, value, "avatar").await?;
+                Some(config_value.as_str())
+            }
+            _ => Some(value),
+        };
+        match key.is_synced() {
+            true => self.set_config_ex(Nosync, *key, value).await,
+            false => Ok(()),
+        }
+    }
+
     fn check_config(key: Config, value: Option<&str>) -> Result<()> {
         match key {
             Config::Socks5Enabled
@@ -565,15 +588,24 @@ impl Context {
                     .execute("UPDATE contacts SET selfavatar_sent=0;", ())
                     .await?;
                 match value {
-                    Some(value) => {
-                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
+                    Some(path) => {
+                        let mut blob = BlobObject::new_from_path(self, path.as_ref()).await?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql
                             .set_raw_config(key.as_ref(), Some(blob.as_name()))
                             .await?;
+                        if sync {
+                            let buf = fs::read(blob.to_abs_path()).await?;
+                            better_value = base64::engine::general_purpose::STANDARD.encode(buf);
+                            value = Some(&better_value);
+                        }
                     }
                     None => {
                         self.sql.set_raw_config(key.as_ref(), None).await?;
+                        if sync {
+                            better_value = String::new();
+                            value = Some(&better_value);
+                        }
                     }
                 }
                 self.emit_event(EventType::SelfavatarChanged);
@@ -747,7 +779,7 @@ mod tests {
 
     use super::*;
     use crate::constants;
-    use crate::test_utils::{sync, TestContext};
+    use crate::test_utils::{sync, TestContext, TestContextManager};
 
     #[test]
     fn test_to_string() {
@@ -941,14 +973,95 @@ mod tests {
         assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
 
         // Usual sync scenario.
+        async fn test_config_str(
+            alice0: &TestContext,
+            alice1: &TestContext,
+            key: Config,
+            val: &str,
+        ) -> Result<()> {
+            alice0.set_config(key, Some(val)).await?;
+            sync(alice0, alice1).await;
+            assert_eq!(alice1.get_config(key).await?, Some(val.to_string()));
+            Ok(())
+        }
+        test_config_str(&alice0, &alice1, Config::Displayname, "Alice Sync").await?;
+        test_config_str(&alice0, &alice1, Config::Selfstatus, "My status").await?;
+
+        assert!(alice0.get_config(Config::Selfavatar).await?.is_none());
+        let file = alice0.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        tokio::fs::write(&file, bytes).await?;
         alice0
-            .set_config(Config::Displayname, Some("Alice Sync"))
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
         sync(&alice0, &alice1).await;
+        assert!(alice1
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".png"))
+            .is_some());
+        alice0.set_config(Config::Selfavatar, None).await?;
+        sync(&alice0, &alice1).await;
+        assert!(alice1.get_config(Config::Selfavatar).await?.is_none());
+
+        Ok(())
+    }
+
+    /// Sync message mustn't be sent if self-{status,avatar} is changed by a self-sent message.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_no_sync_on_self_sent_msg() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice0 = &tcm.alice().await;
+        let alice1 = &tcm.alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let status = "Synced via usual message";
+        alice0.set_config(Config::Selfstatus, Some(status)).await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let status1 = "Synced via sync message";
+        alice1.set_config(Config::Selfstatus, Some(status1)).await?;
+        tcm.send_recv(alice0, alice1, "hi Alice!").await;
         assert_eq!(
-            alice1.get_config(Config::Displayname).await?,
-            Some("Alice Sync".to_string())
+            alice1.get_config(Config::Selfstatus).await?,
+            Some(status.to_string())
         );
+        sync(alice1, alice0).await;
+        assert_eq!(
+            alice0.get_config(Config::Selfstatus).await?,
+            Some(status1.to_string())
+        );
+
+        // Need a chat with another contact to send self-avatar.
+        let bob = &tcm.bob().await;
+        let a0b_chat_id = tcm.send_recv_accept(bob, alice0, "hi").await.chat_id;
+        let file = alice0.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        tokio::fs::write(&file, bytes).await?;
+        alice0
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let file = alice1.dir.path().join("avatar.jpg");
+        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        tokio::fs::write(&file, bytes).await?;
+        alice1
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        let sent_msg = alice0.send_text(a0b_chat_id, "hi").await;
+        alice1.recv_msg(&sent_msg).await;
+        assert!(alice1
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".png"))
+            .is_some());
+        sync(alice1, alice0).await;
+        assert!(alice0
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".jpg"))
+            .is_some());
 
         Ok(())
     }

src/configure.rs

@@ -25,7 +25,7 @@ use tokio::task;
 use crate::config::{self, Config};
 use crate::contact::addr_cmp;
 use crate::context::Context;
-use crate::imap::Imap;
+use crate::imap::{session::Session as ImapSession, Imap};
 use crate::log::LogExt;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
@@ -395,7 +395,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     // Configure IMAP
 
-    let mut imap: Option<Imap> = None;
+    let mut imap: Option<(Imap, ImapSession)> = None;
     let imap_servers: Vec<&ServerParams> = servers
         .iter()
         .filter(|params| params.protocol == Protocol::Imap)
@@ -433,7 +433,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
         );
     }
-    let mut imap = match imap {
+    let (mut imap, mut imap_session) = match imap {
         Some(imap) => imap,
         None => bail!(nicer_configuration_error(ctx, errors).await),
     };
@@ -454,9 +454,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let create_mvbox = ctx.should_watch_mvbox().await?;
 
-    imap.configure_folders(ctx, create_mvbox).await?;
+    imap.configure_folders(ctx, &mut imap_session, create_mvbox)
+        .await?;
 
-    imap.select_with_uidvalidity(ctx, "INBOX")
+    imap_session
+        .select_with_uidvalidity(ctx, "INBOX")
         .await
         .context("could not read INBOX status")?;
 
@@ -576,7 +578,7 @@ async fn try_imap_one_param(
     socks5_config: &Option<Socks5Config>,
     addr: &str,
     provider_strict_tls: bool,
-) -> Result<Imap, ConfigurationError> {
+) -> Result<(Imap, ImapSession), ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
         param.user,
@@ -614,9 +616,9 @@ async fn try_imap_one_param(
                 msg: format!("{err:#}"),
             })
         }
-        Ok(()) => {
+        Ok(session) => {
             info!(context, "success: {}", inf);
-            Ok(imap)
+            Ok((imap, session))
         }
     }
 }

src/contact.rs

@@ -6,7 +6,7 @@ use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::UNIX_EPOCH;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
@@ -36,7 +36,7 @@ use crate::sql::{self, params_iter};
 use crate::sync::{self, Sync::*};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
-    EmailAddress,
+    EmailAddress, SystemTime,
 };
 use crate::{chat, stock_str};
 
@@ -1537,7 +1537,7 @@ WHERE type=? AND id IN (
 /// as profile images can be set only by receiving messages, this should be always the case, however.
 ///
 /// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
-/// this typically happens if we see message with our own profile image, sent from another device.
+/// this typically happens if we see message with our own profile image.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: ContactId,
@@ -1550,7 +1550,7 @@ pub(crate) async fn set_profile_image(
             if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
-                        .set_config_internal(Config::Selfavatar, Some(profile_image))
+                        .set_config_ex(Nosync, Config::Selfavatar, Some(profile_image))
                         .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar.");
@@ -1564,7 +1564,7 @@ pub(crate) async fn set_profile_image(
             if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
-                        .set_config_internal(Config::Selfavatar, None)
+                        .set_config_ex(Nosync, Config::Selfavatar, None)
                         .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar deletion.");
@@ -1597,7 +1597,7 @@ pub(crate) async fn set_status(
     if contact_id == ContactId::SELF {
         if encrypted && has_chat_version {
             context
-                .set_config_internal(Config::Selfstatus, Some(&status))
+                .set_config_ex(Nosync, Config::Selfstatus, Some(&status))
                 .await?;
         }
     } else {

src/context.rs

@@ -20,14 +20,17 @@ use crate::config::Config;
 use crate::constants::{
     self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR,
 };
-use crate::contact::Contact;
+use crate::contact::{Contact, ContactId};
 use crate::debug_logging::DebugLogging;
+use crate::download::DownloadState;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::imap::{FolderMeaning, Imap, ServerMetadata};
 use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _};
 use crate::login_param::LoginParam;
 use crate::message::{self, Message, MessageState, MsgId, Viewtype};
+use crate::param::{Param, Params};
 use crate::peerstate::Peerstate;
+use crate::push::PushSubscriber;
 use crate::quota::QuotaInfo;
 use crate::scheduler::{convert_folder_meaning, SchedulerState};
 use crate::sql::Sql;
@@ -84,6 +87,8 @@ pub struct ContextBuilder {
     events: Events,
     stock_strings: StockStrings,
     password: Option<String>,
+
+    push_subscriber: Option<PushSubscriber>,
 }
 
 impl ContextBuilder {
@@ -99,6 +104,7 @@ impl ContextBuilder {
             events: Events::new(),
             stock_strings: StockStrings::new(),
             password: None,
+            push_subscriber: None,
         }
     }
 
@@ -153,11 +159,32 @@ impl ContextBuilder {
         self
     }
 
-    /// Opens the [`Context`].
+    /// Sets push subscriber.
+    pub(crate) fn with_push_subscriber(mut self, push_subscriber: PushSubscriber) -> Self {
+        self.push_subscriber = Some(push_subscriber);
+        self
+    }
+
+    /// Builds the [`Context`] without opening it.
+    pub async fn build(self) -> Result<Context> {
+        let push_subscriber = self.push_subscriber.unwrap_or_default();
+        let context = Context::new_closed(
+            &self.dbfile,
+            self.id,
+            self.events,
+            self.stock_strings,
+            push_subscriber,
+        )
+        .await?;
+        Ok(context)
+    }
+
+    /// Builds the [`Context`] and opens it.
+    ///
+    /// Returns error if context cannot be opened with the given passphrase.
     pub async fn open(self) -> Result<Context> {
-        let context =
-            Context::new_closed(&self.dbfile, self.id, self.events, self.stock_strings).await?;
-        let password = self.password.unwrap_or_default();
+        let password = self.password.clone().unwrap_or_default();
+        let context = self.build().await?;
         match context.open(password).await? {
             true => Ok(context),
             false => bail!("database could not be decrypted, incorrect or missing password"),
@@ -253,6 +280,13 @@ pub struct InnerContext {
     /// Standard RwLock instead of [`tokio::sync::RwLock`] is used
     /// because the lock is used from synchronous [`Context::emit_event`].
     pub(crate) debug_logging: std::sync::RwLock<Option<DebugLogging>>,
+
+    /// Push subscriber to store device token
+    /// and register for heartbeat notifications.
+    pub(crate) push_subscriber: PushSubscriber,
+
+    /// True if account has subscribed to push notifications via IMAP.
+    pub(crate) push_subscribed: AtomicBool,
 }
 
 /// The state of ongoing process.
@@ -298,7 +332,8 @@ impl Context {
         events: Events,
         stock_strings: StockStrings,
     ) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id, events, stock_strings).await?;
+        let context =
+            Self::new_closed(dbfile, id, events, stock_strings, Default::default()).await?;
 
         // Open the database if is not encrypted.
         if context.check_passphrase("".to_string()).await? {
@@ -313,6 +348,7 @@ impl Context {
         id: u32,
         events: Events,
         stockstrings: StockStrings,
+        push_subscriber: PushSubscriber,
     ) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -321,7 +357,14 @@ impl Context {
         if !blobdir.exists() {
             tokio::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile.into(), blobdir, id, events, stockstrings)?;
+        let context = Context::with_blobdir(
+            dbfile.into(),
+            blobdir,
+            id,
+            events,
+            stockstrings,
+            push_subscriber,
+        )?;
         Ok(context)
     }
 
@@ -364,6 +407,7 @@ impl Context {
         id: u32,
         events: Events,
         stockstrings: StockStrings,
+        push_subscriber: PushSubscriber,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir(),
@@ -398,6 +442,8 @@ impl Context {
             last_full_folder_scan: Mutex::new(None),
             last_error: std::sync::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
+            push_subscriber,
+            push_subscribed: AtomicBool::new(false),
         };
 
         let ctx = Context {
@@ -461,13 +507,13 @@ impl Context {
 
         // connection
         let mut connection = Imap::new_configured(self, channel::bounded(1).1).await?;
-        connection.prepare(self).await?;
+        let mut session = connection.prepare(self).await?;
 
         // fetch imap folders
         for folder_meaning in [FolderMeaning::Inbox, FolderMeaning::Mvbox] {
             let (_, watch_folder) = convert_folder_meaning(self, folder_meaning).await?;
             connection
-                .fetch_move_delete(self, &watch_folder, folder_meaning)
+                .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
                 .await?;
         }
 
@@ -484,7 +530,7 @@ impl Context {
         };
 
         if quota_needs_update {
-            if let Err(err) = self.update_recent_quota(&mut connection).await {
+            if let Err(err) = self.update_recent_quota(&mut session).await {
                 warn!(self, "Failed to update quota: {err:#}.");
             }
         }
@@ -715,7 +761,7 @@ impl Context {
         );
         res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
-        res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
+        res.insert("displayname", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
@@ -877,6 +923,16 @@ impl Context {
     }
 
     async fn get_self_report(&self) -> Result<String> {
+        #[derive(Default)]
+        struct ChatNumbers {
+            protected: u32,
+            protection_broken: u32,
+            opportunistic_dc: u32,
+            opportunistic_mua: u32,
+            unencrypted_dc: u32,
+            unencrypted_mua: u32,
+        }
+
         let mut res = String::new();
         res += &format!("core_version {}\n", get_version_str());
 
@@ -904,6 +960,76 @@ impl Context {
         let key_created = secret_key.created_at().timestamp();
         res += &format!("key_created {}\n", key_created);
 
+        // how many of the chats active in the last months are:
+        // - protected
+        // - protection-broken
+        // - opportunistic-encrypted and the contact uses Delta Chat
+        // - opportunistic-encrypted and the contact uses a classical MUA
+        // - unencrypted and the contact uses Delta Chat
+        // - unencrypted and the contact uses a classical MUA
+        let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3);
+        let chats = self
+            .sql
+            .query_map(
+                "SELECT c.protected, m.param, m.msgrmsg
+                    FROM chats c
+                    JOIN msgs m
+                        ON c.id=m.chat_id
+                        AND m.id=(
+                                SELECT id
+                                FROM msgs
+                                WHERE chat_id=c.id
+                                AND hidden=0
+                                AND download_state=?
+                                AND to_id!=?
+                                ORDER BY timestamp DESC, id DESC LIMIT 1)
+                    WHERE c.id>9
+                    AND (c.blocked=0 OR c.blocked=2)
+                    AND IFNULL(m.timestamp,c.created_timestamp) > ?
+                    GROUP BY c.id",
+                (DownloadState::Done, ContactId::INFO, three_months_ago),
+                |row| {
+                    let protected: ProtectionStatus = row.get(0)?;
+                    let message_param: Params =
+                        row.get::<_, String>(1)?.parse().unwrap_or_default();
+                    let is_dc_message: bool = row.get(2)?;
+                    Ok((protected, message_param, is_dc_message))
+                },
+                |rows| {
+                    let mut chats = ChatNumbers::default();
+                    for row in rows {
+                        let (protected, message_param, is_dc_message) = row?;
+                        let encrypted = message_param
+                            .get_bool(Param::GuaranteeE2ee)
+                            .unwrap_or(false);
+
+                        if protected == ProtectionStatus::Protected {
+                            chats.protected += 1;
+                        } else if protected == ProtectionStatus::ProtectionBroken {
+                            chats.protection_broken += 1;
+                        } else if encrypted {
+                            if is_dc_message {
+                                chats.opportunistic_dc += 1;
+                            } else {
+                                chats.opportunistic_mua += 1;
+                            }
+                        } else if is_dc_message {
+                            chats.unencrypted_dc += 1;
+                        } else {
+                            chats.unencrypted_mua += 1;
+                        }
+                    }
+                    Ok(chats)
+                },
+            )
+            .await?;
+        res += &format!("chats_protected {}\n", chats.protected);
+        res += &format!("chats_protection_broken {}\n", chats.protection_broken);
+        res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc);
+        res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua);
+        res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc);
+        res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua);
+
         let self_reporting_id = match self.get_config(Config::SelfReportingId).await? {
             Some(id) => id,
             None => {
@@ -1217,7 +1343,7 @@ pub fn get_version_str() -> &'static str {
 
 #[cfg(test)]
 mod tests {
-    use std::time::{Duration, SystemTime};
+    use std::time::Duration;
 
     use anyhow::Context as _;
     use strum::IntoEnumIterator;
@@ -1234,7 +1360,7 @@ mod tests {
     use crate::mimeparser::SystemMessage;
     use crate::receive_imf::receive_imf;
     use crate::test_utils::{get_chat_msg, TestContext};
-    use crate::tools::create_outgoing_rfc724_mid;
+    use crate::tools::{create_outgoing_rfc724_mid, SystemTime};
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_db() -> Result<()> {
@@ -1419,7 +1545,14 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new(), StockStrings::new());
+        let res = Context::with_blobdir(
+            dbfile,
+            blobdir,
+            1,
+            Events::new(),
+            StockStrings::new(),
+            Default::default(),
+        );
         assert!(res.is_err());
     }
 
@@ -1428,7 +1561,14 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new(), StockStrings::new());
+        let res = Context::with_blobdir(
+            dbfile,
+            blobdir,
+            1,
+            Events::new(),
+            StockStrings::new(),
+            Default::default(),
+        );
         assert!(res.is_err());
     }
 
@@ -1651,16 +1791,18 @@ mod tests {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
-        let id = 1;
-        let context = Context::new_closed(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile.clone())
+            .with_id(1)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);
         assert_eq!(context.is_open().await, true);
         drop(context);
 
-        let id = 2;
-        let context = Context::new(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile)
+            .with_id(2)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.is_open().await, false);
@@ -1676,8 +1818,9 @@ mod tests {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
-        let id = 1;
-        let context = Context::new_closed(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile)
+            .with_id(1)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);

src/download.rs

@@ -3,13 +3,13 @@
 use std::cmp::max;
 use std::collections::BTreeMap;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 
 use crate::config::Config;
 use crate::context::Context;
-use crate::imap::{Imap, ImapActionResult};
+use crate::imap::session::Session;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::tools::time;
@@ -129,9 +129,11 @@ impl Message {
 /// Actually download a message partially downloaded before.
 ///
 /// Most messages are downloaded automatically on fetch instead.
-pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Imap) -> Result<()> {
-    imap.prepare(context).await?;
-
+pub(crate) async fn download_msg(
+    context: &Context,
+    msg_id: MsgId,
+    session: &mut Session,
+) -> Result<()> {
     let msg = Message::load_from_db(context, msg_id).await?;
     let row = context
         .sql
@@ -152,7 +154,7 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
         return Err(anyhow!("Call download_full() again to try over."));
     };
 
-    match imap
+    session
         .fetch_single_msg(
             context,
             &server_folder,
@@ -160,16 +162,11 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
             server_uid,
             msg.rfc724_mid.clone(),
         )
-        .await
-    {
-        ImapActionResult::RetryLater | ImapActionResult::Failed => {
-            Err(anyhow!("Call download_full() again to try over."))
-        }
-        ImapActionResult::Success => Ok(()),
-    }
+        .await?;
+    Ok(())
 }
 
-impl Imap {
+impl Session {
     /// Download a single message and pipe it to receive_imf().
     ///
     /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
@@ -181,20 +178,19 @@ impl Imap {
         uidvalidity: u32,
         uid: u32,
         rfc724_mid: String,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
+    ) -> Result<()> {
+        if uid == 0 {
+            bail!("Attempt to fetch UID 0");
         }
 
+        self.select_folder(context, Some(folder)).await?;
+
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
 
         let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
         uid_message_ids.insert(uid, rfc724_mid);
-        let (last_uid, _received) = match self
+        let (last_uid, _received) = self
             .fetch_many_msgs(
                 context,
                 folder,
@@ -204,16 +200,11 @@ impl Imap {
                 false,
                 false,
             )
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return ImapActionResult::Failed,
-        };
+            .await?;
         if last_uid.is_none() {
-            ImapActionResult::Failed
-        } else {
-            ImapActionResult::Success
+            bail!("Failed to fetch UID {uid}");
         }
+        Ok(())
     }
 }
 

src/ephemeral.rs

@@ -67,7 +67,7 @@ use std::collections::BTreeSet;
 use std::convert::{TryFrom, TryInto};
 use std::num::ParseIntError;
 use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, UNIX_EPOCH};
 
 use anyhow::{ensure, Result};
 use async_channel::Receiver;
@@ -85,7 +85,7 @@ use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::sql::{self, params_iter};
 use crate::stock_str;
-use crate::tools::{duration_to_str, time};
+use crate::tools::{duration_to_str, time, SystemTime};
 
 /// Ephemeral timer value.
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]

src/imap/capabilities.rs

@@ -25,6 +25,13 @@ pub(crate) struct Capabilities {
     /// <https://tools.ietf.org/html/rfc5464>
     pub can_metadata: bool,
 
+    /// True if the server supports XDELTAPUSH capability.
+    /// This capability means setting /private/devicetoken IMAP METADATA
+    /// on the INBOX results in new mail notifications
+    /// via notifications.delta.chat service.
+    /// This is supported by <https://github.com/deltachat/chatmail>
+    pub can_push: bool,
+
     /// Server ID if the server supports ID capability.
     pub server_id: Option<HashMap<String, String>>,
 }

src/imap/client.rs

@@ -60,6 +60,7 @@ async fn determine_capabilities(
         can_check_quota: caps.has_str("QUOTA"),
         can_condstore: caps.has_str("CONDSTORE"),
         can_metadata: caps.has_str("METADATA"),
+        can_push: caps.has_str("XDELTAPUSH"),
         server_id,
     };
     Ok(capabilities)

src/imap/idle.rs

@@ -4,13 +4,12 @@ use anyhow::{bail, Context as _, Result};
 use async_channel::Receiver;
 use async_imap::extensions::idle::IdleResponse;
 use futures_lite::FutureExt;
+use tokio::time::timeout;
 
 use super::session::Session;
 use super::Imap;
-use crate::config::Config;
 use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
-use crate::log::LogExt;
 use crate::tools::{self, time_elapsed};
 
 /// Timeout after which IDLE is finished
@@ -98,97 +97,38 @@ impl Session {
 }
 
 impl Imap {
+    /// Idle using polling.
     pub(crate) async fn fake_idle(
         &mut self,
         context: &Context,
-        watch_folder: Option<String>,
+        session: &mut Session,
+        watch_folder: String,
         folder_meaning: FolderMeaning,
-    ) {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
-
+    ) -> Result<()> {
         let fake_idle_start_time = tools::Time::now();
 
-        // Do not poll, just wait for an interrupt when no folder is passed in.
-        let watch_folder = if let Some(watch_folder) = watch_folder {
-            watch_folder
-        } else {
-            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
-            self.idle_interrupt_receiver.recv().await.ok();
-            return;
-        };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        const TIMEOUT_INIT_MS: u64 = 60_000;
-        let mut timeout_ms: u64 = TIMEOUT_INIT_MS;
-        enum Event {
-            Tick,
-            Interrupt,
-        }
-        // loop until we are interrupted or if we fetched something
+        // Loop until we are interrupted or until we fetch something.
         loop {
-            use futures::future::FutureExt;
-            use rand::Rng;
-
-            let mut interval = tokio::time::interval(Duration::from_millis(timeout_ms));
-            timeout_ms = timeout_ms
-                .saturating_add(rand::thread_rng().gen_range((timeout_ms / 2)..=timeout_ms));
-            interval.tick().await; // The first tick completes immediately.
-            match interval
-                .tick()
-                .map(|_| Event::Tick)
-                .race(
-                    self.idle_interrupt_receiver
-                        .recv()
-                        .map(|_| Event::Interrupt),
-                )
-                .await
-            {
-                Event::Tick => {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.prepare(context).await {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if let Some(session) = &self.session {
-                        if session.can_idle()
-                            && !context
-                                .get_config_bool(Config::DisableIdle)
-                                .await
-                                .context("Failed to get disable_idle config")
-                                .log_err(context)
-                                .unwrap_or_default()
-                        {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
+            match timeout(Duration::from_secs(60), self.idle_interrupt_receiver.recv()).await {
+                Err(_) => {
+                    // Let's see if fetching messages results
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-                    match self
-                        .fetch_new_messages(context, &watch_folder, folder_meaning, false)
-                        .await
-                    {
-                        Ok(res) => {
-                            info!(context, "fetch_new_messages returned {:?}", res);
-                            timeout_ms = TIMEOUT_INIT_MS;
-                            if res {
-                                break;
-                            }
-                        }
-                        Err(err) => {
-                            error!(context, "could not fetch from folder: {:#}", err);
-                            self.trigger_reconnect(context);
-                        }
+                    let res = self
+                        .fetch_new_messages(context, session, &watch_folder, folder_meaning, false)
+                        .await?;
+
+                    info!(context, "fetch_new_messages returned {:?}", res);
+
+                    if res {
+                        break;
                     }
                 }
-                Event::Interrupt => {
-                    info!(context, "Fake IDLE interrupted");
+                Ok(_) => {
+                    info!(context, "Fake IDLE interrupted.");
                     break;
                 }
             }
@@ -199,5 +139,6 @@ impl Imap {
             "IMAP-fake-IDLE done after {:.4}s",
             time_elapsed(&fake_idle_start_time).as_millis() as f64 / 1000.,
         );
+        Ok(())
     }
 }

src/imap/scan_folders.rs

@@ -1,18 +1,21 @@
 use std::collections::BTreeMap;
 
 use anyhow::{Context as _, Result};
-use futures::TryStreamExt;
 
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
-use crate::imap::Imap;
+use crate::imap::{session::Session, Imap};
 use crate::log::LogExt;
 use crate::tools::{self, time_elapsed};
 use crate::{context::Context, imap::FolderMeaning};
 
 impl Imap {
     /// Returns true if folders were scanned, false if scanning was postponed.
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
+    pub(crate) async fn scan_folders(
+        &mut self,
+        context: &Context,
+        session: &mut Session,
+    ) -> Result<bool> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
@@ -27,8 +30,7 @@ impl Imap {
         }
         info!(context, "Starting full folder scan");
 
-        self.prepare(context).await?;
-        let folders = self.list_folders().await?;
+        let folders = session.list_folders().await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
@@ -64,18 +66,16 @@ impl Imap {
                 && folder_meaning != FolderMeaning::Drafts
                 && folder_meaning != FolderMeaning::Trash
             {
-                let session = self.session.as_mut().context("no session")?;
                 // Drain leftover unsolicited EXISTS messages
                 session.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_move_delete(context, folder.name(), folder_meaning)
+                    self.fetch_move_delete(context, session, folder.name(), folder_meaning)
                         .await
                         .context("Can't fetch new msgs in scanned folder")
                         .log_err(context)
                         .ok();
 
-                    let session = self.session.as_mut().context("no session")?;
                     // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
                     if !session.server_sent_unsolicited_exists(context)? {
                         break;
@@ -97,18 +97,6 @@ impl Imap {
         last_scan.replace(tools::Time::now());
         Ok(true)
     }
-
-    /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
-        let session = self.session.as_mut();
-        let session = session.context("No IMAP connection")?;
-        let list = session
-            .list(Some(""), Some("*"))
-            .await?
-            .try_collect()
-            .await?;
-        Ok(list)
-    }
 }
 
 pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {

src/imap/select_folder.rs

@@ -3,6 +3,7 @@
 use anyhow::Context as _;
 
 use super::session::Session as ImapSession;
+use super::{get_uid_next, get_uidvalidity, set_modseq, set_uid_next, set_uidvalidity};
 use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
@@ -51,7 +52,7 @@ impl ImapSession {
     /// Selects a folder, possibly updating uid_validity and, if needed,
     /// expunging the folder to remove delete-marked messages.
     /// Returns whether a new folder was selected.
-    pub(super) async fn select_folder(
+    pub(crate) async fn select_folder(
         &mut self,
         context: &Context,
         folder: Option<&str>,
@@ -127,10 +128,135 @@ impl ImapSession {
             },
         }
     }
+
+    /// Selects a folder and takes care of UIDVALIDITY changes.
+    ///
+    /// When selecting a folder for the first time, sets the uid_next to the current
+    /// mailbox.uid_next so that no old emails are fetched.
+    ///
+    /// Returns Result<new_emails> (i.e. whether new emails arrived),
+    /// if in doubt, returns new_emails=true so emails are fetched.
+    pub(crate) async fn select_with_uidvalidity(
+        &mut self,
+        context: &Context,
+        folder: &str,
+    ) -> Result<bool> {
+        let newly_selected = self
+            .select_or_create_folder(context, folder)
+            .await
+            .with_context(|| format!("failed to select or create folder {folder}"))?;
+        let mailbox = self
+            .selected_mailbox
+            .as_mut()
+            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
+
+        let old_uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
+        let old_uid_next = get_uid_next(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+
+        let new_uid_validity = mailbox
+            .uid_validity
+            .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
+        let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
+            Some(uid_next)
+        } else {
+            warn!(
+                context,
+                "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
+            );
+
+            // RFC 3501 says STATUS command SHOULD NOT be used
+            // on the currently selected mailbox because the same
+            // information can be obtained by other means,
+            // such as reading SELECT response.
+            //
+            // However, it also says that UIDNEXT is REQUIRED
+            // in the SELECT response and if we are here,
+            // it is actually not returned.
+            //
+            // In particular, Winmail Pro Mail Server 5.1.0616
+            // never returns UIDNEXT in SELECT response,
+            // but responds to "STATUS INBOX (UIDNEXT)" command.
+            let status = self
+                .inner
+                .status(folder, "(UIDNEXT)")
+                .await
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+
+            if status.uid_next.is_none() {
+                // This happens with mail.163.com as of 2023-11-26.
+                // It does not return UIDNEXT on SELECT and returns invalid
+                // `* STATUS "INBOX" ()` response on explicit request for UIDNEXT.
+                warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT.");
+            }
+            status.uid_next
+        };
+        mailbox.uid_next = new_uid_next;
+
+        if new_uid_validity == old_uid_validity {
+            let new_emails = if newly_selected == NewlySelected::No {
+                // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next
+                // was not updated and may contain an incorrect value. So, just return true so that
+                // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
+                // new messages is only one command, just as a SELECT command)
+                true
+            } else if let Some(new_uid_next) = new_uid_next {
+                if new_uid_next < old_uid_next {
+                    warn!(
+                        context,
+                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                    );
+                    set_uid_next(context, folder, new_uid_next).await?;
+                    context.schedule_resync().await?;
+                }
+                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
+            } else {
+                // We have no UIDNEXT and if in doubt, return true.
+                true
+            };
+
+            return Ok(new_emails);
+        }
+
+        // UIDVALIDITY is modified, reset highest seen MODSEQ.
+        set_modseq(context, folder, 0).await?;
+
+        // ==============  uid_validity has changed or is being set the first time.  ==============
+
+        let new_uid_next = new_uid_next.unwrap_or_default();
+        set_uid_next(context, folder, new_uid_next).await?;
+        set_uidvalidity(context, folder, new_uid_validity).await?;
+
+        // Collect garbage entries in `imap` table.
+        context
+            .sql
+            .execute(
+                "DELETE FROM imap WHERE folder=? AND uidvalidity!=?",
+                (&folder, new_uid_validity),
+            )
+            .await?;
+
+        if old_uid_validity != 0 || old_uid_next != 0 {
+            context.schedule_resync().await?;
+        }
+        info!(
+            context,
+            "uid/validity change folder {}: new {}/{} previous {}/{}.",
+            folder,
+            new_uid_next,
+            new_uid_validity,
+            old_uid_next,
+            old_uid_validity,
+        );
+        Ok(false)
+    }
 }
 
 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
-pub(super) enum NewlySelected {
+pub(crate) enum NewlySelected {
     /// The folder was newly selected during this call to select_folder().
     Yes,
     /// No SELECT command was run because the folder already was selected

src/imap/session.rs

@@ -1,11 +1,32 @@
+use std::cmp;
+use std::collections::BTreeMap;
 use std::ops::{Deref, DerefMut};
 
+use anyhow::{Context as _, Result};
 use async_imap::types::Mailbox;
 use async_imap::Session as ImapSession;
+use futures::TryStreamExt;
 
+use crate::constants::DC_FETCH_EXISTING_MSGS_COUNT;
 use crate::imap::capabilities::Capabilities;
 use crate::net::session::SessionStream;
 
+/// Prefetch:
+/// - Message-ID to check if we already have the message.
+/// - In-Reply-To and References to check if message is a reply to chat message.
+/// - Chat-Version to check if a message is a chat message
+/// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
+///   not necessarily sent by Delta Chat.
+const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
+                              MESSAGE-ID \
+                              DATE \
+                              X-MICROSOFT-ORIGINAL-MESSAGE-ID \
+                              FROM \
+                              IN-REPLY-TO REFERENCES \
+                              CHAT-VERSION \
+                              AUTOCRYPT-SETUP-MESSAGE\
+                              )])";
+
 #[derive(Debug)]
 pub(crate) struct Session {
     pub(super) inner: ImapSession<Box<dyn SessionStream>>,
@@ -68,4 +89,75 @@ impl Session {
     pub fn can_metadata(&self) -> bool {
         self.capabilities.can_metadata
     }
+
+    pub fn can_push(&self) -> bool {
+        self.capabilities.can_push
+    }
+
+    /// Returns the names of all folders on the IMAP server.
+    pub async fn list_folders(&mut self) -> Result<Vec<async_imap::types::Name>> {
+        let list = self.list(Some(""), Some("*")).await?.try_collect().await?;
+        Ok(list)
+    }
+
+    /// Prefetch all messages greater than or equal to `uid_next`. Returns a list of fetch results
+    /// in the order of ascending delivery time to the server (INTERNALDATE).
+    pub(crate) async fn prefetch(
+        &mut self,
+        uid_next: u32,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        // fetch messages with larger UID than the last one seen
+        let set = format!("{uid_next}:*");
+        let mut list = self
+            .uid_fetch(set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                // If the mailbox is not empty, results always include
+                // at least one UID, even if last_seen_uid+1 is past
+                // the last UID in the mailbox.  It happens because
+                // uid:* is interpreted the same way as *:uid.
+                // See <https://tools.ietf.org/html/rfc3501#page-61> for
+                // standard reference. Therefore, sometimes we receive
+                // already seen messages and have to filter them out.
+                if msg_uid >= uid_next {
+                    msgs.insert((msg.internal_date(), msg_uid), msg);
+                }
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
+
+    /// Like prefetch(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
+    pub(crate) async fn prefetch_existing_msgs(
+        &mut self,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        let exists: i64 = {
+            let mailbox = self.selected_mailbox.as_ref().context("no mailbox")?;
+            mailbox.exists.into()
+        };
+
+        // Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
+        // Sequence numbers are sequential. If there are 1000 messages in the inbox,
+        // we can fetch the sequence numbers 900-1000 and get the last 100 messages.
+        let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT + 1);
+        let set = format!("{first}:{exists}");
+        let mut list = self
+            .fetch(&set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP Could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert((msg.internal_date(), msg_uid), msg);
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
 }

src/imex.rs

@@ -389,11 +389,11 @@ async fn imex_inner(
 
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
-        if e2ee::ensure_secret_key_exists(context).await.is_err() {
-            bail!("Cannot create private key or private key not available.");
-        } else {
-            create_folder(context, &path).await?;
-        }
+        e2ee::ensure_secret_key_exists(context)
+            .await
+            .context("Cannot create private key or private key not available")?;
+
+        create_folder(context, &path).await?;
     }
 
     match what {

src/key.rs

@@ -79,7 +79,7 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 }
 
 pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPublicKey> {
-    match context
+    let public_key = context
         .sql
         .query_row_optional(
             "SELECT public_key
@@ -91,8 +91,8 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
                 Ok(bytes)
             },
         )
-        .await?
-    {
+        .await?;
+    match public_key {
         Some(bytes) => SignedPublicKey::from_slice(&bytes),
         None => {
             let keypair = generate_keypair(context).await?;
@@ -102,7 +102,7 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
 }
 
 pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecretKey> {
-    match context
+    let private_key = context
         .sql
         .query_row_optional(
             "SELECT private_key
@@ -114,8 +114,8 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
                 Ok(bytes)
             },
         )
-        .await?
-    {
+        .await?;
+    match private_key {
         Some(bytes) => SignedSecretKey::from_slice(&bytes),
         None => {
             let keypair = generate_keypair(context).await?;
@@ -255,7 +255,7 @@ pub(crate) async fn load_keypair(
 
 /// Use of a key pair for encryption or decryption.
 ///
-/// This is used by [store_self_keypair] to know what kind of key is
+/// This is used by `store_self_keypair` to know what kind of key is
 /// being saved.
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum KeyPairUse {
@@ -277,7 +277,7 @@ pub enum KeyPairUse {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub async fn store_self_keypair(
+pub(crate) async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,

src/lib.rs

@@ -98,6 +98,7 @@ mod color;
 pub mod html;
 pub mod net;
 pub mod plaintext;
+mod push;
 pub mod summary;
 
 mod debug_logging;

src/mimefactory.rs

@@ -338,18 +338,11 @@ impl<'a> MimeFactory<'a> {
     fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.is_protected() {
-                    false
-                } else if chat.typ == Chattype::Broadcast {
-                    // encryption may disclose recipients;
-                    // this is probably a worse issue than not opportunistically (!) encrypting
-                    true
-                } else {
-                    self.msg
-                        .param
-                        .get_bool(Param::ForcePlaintext)
-                        .unwrap_or_default()
-                }
+                self.msg
+                    .param
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    || chat.typ == Chattype::Broadcast
             }
             Loaded::Mdn { .. } => false,
         }
@@ -940,7 +933,6 @@ impl<'a> MimeFactory<'a> {
         };
         let command = self.msg.param.get_cmd();
         let mut placeholdertext = None;
-        let mut meta_part = None;
 
         let send_verified_headers = match chat.typ {
             Chattype::Single => true,
@@ -1126,17 +1118,13 @@ impl<'a> MimeFactory<'a> {
 
         if let Some(grpimage) = grpimage {
             info!(context, "setting group image '{}'", grpimage);
-            let mut meta = Message {
-                viewtype: Viewtype::Image,
-                ..Default::default()
-            };
-            meta.param.set(Param::File, grpimage);
-
-            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
-            meta_part = Some(mail);
-            headers
-                .protected
-                .push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            let avatar = build_avatar_file(context, grpimage)
+                .await
+                .context("Cannot attach group image")?;
+            headers.hidden.push(Header::new(
+                "Chat-Group-Avatar".into(),
+                format!("base64:{avatar}"),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
@@ -1260,10 +1248,6 @@ impl<'a> MimeFactory<'a> {
             parts.push(file_part);
         }
 
-        if let Some(meta_part) = meta_part {
-            parts.push(meta_part);
-        }
-
         if let Some(msg_kml_part) = self.get_message_kml_part() {
             parts.push(msg_kml_part);
         }
@@ -1295,7 +1279,7 @@ impl<'a> MimeFactory<'a> {
 
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
-                Some(path) => match build_selfavatar_file(context, &path).await {
+                Some(path) => match build_avatar_file(context, &path).await {
                     Ok(avatar) => headers.hidden.push(Header::new(
                         "Chat-User-Avatar".into(),
                         format!("base64:{avatar}"),
@@ -1504,8 +1488,11 @@ async fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-async fn build_selfavatar_file(context: &Context, path: &str) -> Result<String> {
-    let blob = BlobObject::from_path(context, path.as_ref())?;
+async fn build_avatar_file(context: &Context, path: &str) -> Result<String> {
+    let blob = match path.starts_with("$BLOBDIR/") {
+        true => BlobObject::from_name(context, path.to_string())?,
+        false => BlobObject::from_path(context, path.as_ref())?,
+    };
     let body = fs::read(blob.to_abs_path()).await?;
     let encoded_body = wrapped_base64_encode(&body);
     Ok(encoded_body)

src/mimeparser.rs

@@ -8,7 +8,6 @@ use std::pin::Pin;
 use std::str;
 
 use anyhow::{bail, Context as _, Result};
-use base64::Engine as _;
 use deltachat_derive::{FromSql, ToSql};
 use format_flowed::unformat_flowed;
 use lettre_email::mime::{self, Mime};
@@ -266,8 +265,10 @@ impl MimeMessage {
                 for field in &part.headers {
                     let key = field.get_key().to_lowercase();
 
-                    // For now only Chat-User-Avatar can be hidden.
-                    if !headers.contains_key(&key) && key == "chat-user-avatar" {
+                    // For now only avatar headers can be hidden.
+                    if !headers.contains_key(&key)
+                        && (key == "chat-user-avatar" || key == "chat-group-avatar")
+                    {
                         headers.insert(key.to_string(), field.get_value());
                     }
                 }
@@ -722,37 +723,20 @@ impl MimeMessage {
     ) -> Option<AvatarAction> {
         if header_value == "0" {
             Some(AvatarAction::Delete)
-        } else if let Some(avatar) = header_value
+        } else if let Some(base64) = header_value
             .split_ascii_whitespace()
             .collect::<String>()
             .strip_prefix("base64:")
-            .map(|x| base64::engine::general_purpose::STANDARD.decode(x))
         {
-            // Avatar sent directly in the header as base64.
-            if let Ok(decoded_data) = avatar {
-                let extension = if let Ok(format) = image::guess_format(&decoded_data) {
-                    if let Some(ext) = format.extensions_str().first() {
-                        format!(".{ext}")
-                    } else {
-                        String::new()
-                    }
-                } else {
-                    String::new()
-                };
-                match BlobObject::create(context, &format!("avatar{extension}"), &decoded_data)
-                    .await
-                {
-                    Ok(blob) => Some(AvatarAction::Change(blob.as_name().to_string())),
-                    Err(err) => {
-                        warn!(
-                            context,
-                            "Could not save decoded avatar to blob file: {:#}", err
-                        );
-                        None
-                    }
+            match BlobObject::store_from_base64(context, base64, "avatar").await {
+                Ok(path) => Some(AvatarAction::Change(path)),
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Could not decode and save avatar to blob file: {:#}", err,
+                    );
+                    None
                 }
-            } else {
-                None
             }
         } else {
             // Avatar sent in attachment, as previous versions of Delta Chat did.

src/peerstate.rs

@@ -337,7 +337,7 @@ impl Peerstate {
             return;
         }
 
-        if message_time > self.last_seen {
+        if message_time >= self.last_seen {
             self.last_seen = message_time;
             self.last_seen_autocrypt = message_time;
             if (header.prefer_encrypt == EncryptPreference::Mutual
@@ -360,7 +360,7 @@ impl Peerstate {
             return;
         }
 
-        if message_time > self.gossip_timestamp {
+        if message_time >= self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
                 self.gossip_key = Some(gossip_header.public_key.clone());
@@ -995,7 +995,7 @@ mod tests {
         assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
 
         // Same header will be applied in the future.
-        peerstate.apply_header(&header, 400);
+        peerstate.apply_header(&header, 300);
         assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
     }
 }

src/pgp.rs

@@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, Context as _, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -138,8 +138,11 @@ pub struct KeyPair {
 }
 
 /// Create a new key pair.
+///
+/// Both secret and public key consist of signing primary key and encryption subkey
+/// as [described in the Autocrypt standard](https://autocrypt.org/level1.html#openpgp-based-key-data).
 pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Result<KeyPair> {
-    let (secret_key_type, public_key_type) = match keygen_type {
+    let (signing_key_type, encryption_key_type) = match keygen_type {
         KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
         KeyGenType::Rsa4096 => (PgpKeyType::Rsa(4096), PgpKeyType::Rsa(4096)),
         KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
@@ -147,8 +150,8 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
 
     let user_id = format!("<{addr}>");
     let key_params = SecretKeyParamsBuilder::default()
-        .key_type(secret_key_type)
-        .can_create_certificates(true)
+        .key_type(signing_key_type)
+        .can_certify(true)
         .can_sign(true)
         .primary_user_id(user_id)
         .passphrase(None)
@@ -170,41 +173,36 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
         ])
         .subkey(
             SubkeyParamsBuilder::default()
-                .key_type(public_key_type)
+                .key_type(encryption_key_type)
                 .can_encrypt(true)
                 .passphrase(None)
                 .build()
-                .map_err(|e| format_err!("{}", e))
                 .context("failed to build subkey parameters")?,
         )
         .build()
-        .map_err(|e| format_err!("{}", e))
-        .context("invalid key params")?;
-    let key = key_params.generate().context("invalid params")?;
-    let private_key = key
+        .context("failed to build key parameters")?;
+
+    let secret_key = key_params
+        .generate()
+        .context("failed to generate the key")?
         .sign(|| "".into())
-        .map_err(|e| format_err!("{}", e))
         .context("failed to sign secret key")?;
-
-    let public_key = private_key.public_key();
-    let public_key = public_key
-        .sign(&private_key, || "".into())
-        .map_err(|e| format_err!("{}", e))
-        .context("failed to sign public key")?;
-
-    private_key
+    secret_key
         .verify()
-        .map_err(|e| format_err!("{}", e))
-        .context("invalid private key generated")?;
+        .context("invalid secret key generated")?;
+
+    let public_key = secret_key
+        .public_key()
+        .sign(&secret_key, || "".into())
+        .context("failed to sign public key")?;
     public_key
         .verify()
-        .map_err(|e| format_err!("{}", e))
         .context("invalid public key generated")?;
 
     Ok(KeyPair {
         addr,
         public: public_key,
-        secret: private_key,
+        secret: secret_key,
     })
 }
 

src/push.rs

@@ -0,0 +1,115 @@
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+use anyhow::Result;
+use tokio::sync::RwLock;
+
+use crate::context::Context;
+use crate::net::http;
+
+/// Manages subscription to Apple Push Notification services.
+///
+/// This structure is created by account manager and is shared between accounts.
+/// To enable notifications, application should request the device token as described in
+/// <https://developer.apple.com/documentation/usernotifications/registering-your-app-with-apns>
+/// and give it to the account manager, which will forward the token in this structure.
+///
+/// Each account (context) can then retrieve device token
+/// from this structure and give it to the email server.
+/// If email server does not support push notifications,
+/// account can call `subscribe` method
+/// to register device token with the heartbeat
+/// notification provider server as a fallback.
+#[derive(Debug, Clone, Default)]
+pub struct PushSubscriber {
+    inner: Arc<RwLock<PushSubscriberState>>,
+}
+
+impl PushSubscriber {
+    /// Creates new push notification subscriber.
+    pub(crate) fn new() -> Self {
+        Default::default()
+    }
+
+    /// Sets device token for Apple Push Notification service.
+    pub(crate) async fn set_device_token(&mut self, token: &str) {
+        self.inner.write().await.device_token = Some(token.to_string());
+    }
+
+    /// Retrieves device token.
+    ///
+    /// Token may be not available if application is not running on Apple platform,
+    /// failed to register for remote notifications or is in the process of registering.
+    ///
+    /// IMAP loop should periodically check if device token is available
+    /// and send the token to the email server if it supports push notifications.
+    pub(crate) async fn device_token(&self) -> Option<String> {
+        self.inner.read().await.device_token.clone()
+    }
+
+    /// Subscribes for heartbeat notifications with previously set device token.
+    pub(crate) async fn subscribe(&self) -> Result<()> {
+        let mut state = self.inner.write().await;
+
+        if state.heartbeat_subscribed {
+            return Ok(());
+        }
+
+        let Some(ref token) = state.device_token else {
+            return Ok(());
+        };
+
+        let socks5_config = None;
+        let response = http::get_client(socks5_config)?
+            .post("https://notifications.delta.chat/register")
+            .body(format!("{{\"token\":\"{token}\"}}"))
+            .send()
+            .await?;
+
+        let response_status = response.status();
+        if response_status.is_success() {
+            state.heartbeat_subscribed = true;
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn heartbeat_subscribed(&self) -> bool {
+        self.inner.read().await.heartbeat_subscribed
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct PushSubscriberState {
+    /// Device token.
+    device_token: Option<String>,
+
+    /// If subscribed to heartbeat push notifications.
+    heartbeat_subscribed: bool,
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[repr(i8)]
+pub enum NotifyState {
+    /// Not subscribed to push notifications.
+    #[default]
+    NotConnected = 0,
+
+    /// Subscribed to heartbeat push notifications.
+    Heartbeat = 1,
+
+    /// Subscribed to push notifications for new messages.
+    Connected = 2,
+}
+
+impl Context {
+    /// Returns push notification subscriber state.
+    pub async fn push_state(&self) -> NotifyState {
+        if self.push_subscribed.load(Ordering::Relaxed) {
+            NotifyState::Connected
+        } else if self.push_subscriber.heartbeat_subscribed().await {
+            NotifyState::Heartbeat
+        } else {
+            NotifyState::NotConnected
+        }
+    }
+}

src/quota.rs

@@ -10,7 +10,6 @@ use crate::config::Config;
 use crate::context::Context;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::session::Session as ImapSession;
-use crate::imap::Imap;
 use crate::message::{Message, Viewtype};
 use crate::tools;
 use crate::{stock_str, EventType};
@@ -111,13 +110,7 @@ impl Context {
     /// As the message is added only once, the user is not spammed
     /// in case for some providers the quota is always at ~100%
     /// and new space is allocated as needed.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> {
-        if let Err(err) = imap.prepare(self).await {
-            warn!(self, "could not connect: {:#}", err);
-            return Ok(());
-        }
-
-        let session = imap.session.as_mut().context("no session")?;
+    pub(crate) async fn update_recent_quota(&self, session: &mut ImapSession) -> Result<()> {
         let quota = if session.can_check_quota() {
             let folders = get_watched_folders(self).await?;
             get_unique_quota_roots_and_usage(session, folders).await

src/scheduler.rs

@@ -19,7 +19,7 @@ use crate::context::Context;
 use crate::download::{download_msg, DownloadState};
 use crate::ephemeral::{self, delete_expired_imap_messages};
 use crate::events::EventType;
-use crate::imap::{FolderMeaning, Imap};
+use crate::imap::{session::Session, FolderMeaning, Imap};
 use crate::location;
 use crate::log::LogExt;
 use crate::message::MsgId;
@@ -330,7 +330,7 @@ pub(crate) struct Scheduler {
     recently_seen_loop: RecentlySeenLoop,
 }
 
-async fn download_msgs(context: &Context, imap: &mut Imap) -> Result<()> {
+async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
     let msg_ids = context
         .sql
         .query_map(
@@ -349,7 +349,7 @@ async fn download_msgs(context: &Context, imap: &mut Imap) -> Result<()> {
         .await?;
 
     for msg_id in msg_ids {
-        if let Err(err) = download_msg(context, msg_id, imap).await {
+        if let Err(err) = download_msg(context, msg_id, session).await {
             warn!(context, "Failed to download message {msg_id}: {:#}.", err);
 
             // Update download state to failure
@@ -392,84 +392,26 @@ async fn inbox_loop(
             return;
         };
 
+        let mut old_session: Option<Session> = None;
         loop {
-            {
-                // Update quota no more than once a minute.
-                let quota_needs_update = {
-                    let quota = ctx.quota.read().await;
-                    quota
-                        .as_ref()
-                        .filter(|quota| time_elapsed(&quota.modified) > Duration::from_secs(60))
-                        .is_none()
-                };
-
-                if quota_needs_update {
-                    if let Err(err) = ctx.update_recent_quota(&mut connection).await {
-                        warn!(ctx, "Failed to update quota: {:#}.", err);
+            let session = if let Some(session) = old_session.take() {
+                session
+            } else {
+                match connection.prepare(&ctx).await {
+                    Err(err) => {
+                        warn!(ctx, "Failed to prepare connection: {:#}.", err);
+                        continue;
                     }
-                }
-            }
-
-            let resync_requested = ctx.resync_request.swap(false, Ordering::Relaxed);
-            if resync_requested {
-                if let Err(err) = connection.resync_folders(&ctx).await {
-                    warn!(ctx, "Failed to resync folders: {:#}.", err);
-                    ctx.resync_request.store(true, Ordering::Relaxed);
-                }
-            }
-
-            maybe_add_time_based_warnings(&ctx).await;
-
-            match ctx.get_config_i64(Config::LastHousekeeping).await {
-                Ok(last_housekeeping_time) => {
-                    let next_housekeeping_time =
-                        last_housekeeping_time.saturating_add(60 * 60 * 24);
-                    if next_housekeeping_time <= time() {
-                        sql::housekeeping(&ctx).await.log_err(&ctx).ok();
-                    }
-                }
-                Err(err) => {
-                    warn!(ctx, "Failed to get last housekeeping time: {}", err);
+                    Ok(session) => session,
                 }
             };
 
-            match ctx.get_config_bool(Config::FetchedExistingMsgs).await {
-                Ok(fetched_existing_msgs) => {
-                    if !fetched_existing_msgs {
-                        // Consider it done even if we fail.
-                        //
-                        // This operation is not critical enough to retry,
-                        // especially if the error is persistent.
-                        if let Err(err) = ctx
-                            .set_config_internal(
-                                Config::FetchedExistingMsgs,
-                                config::from_bool(true),
-                            )
-                            .await
-                        {
-                            warn!(ctx, "Can't set Config::FetchedExistingMsgs: {:#}", err);
-                        }
-
-                        if let Err(err) = connection.fetch_existing_msgs(&ctx).await {
-                            warn!(ctx, "Failed to fetch existing messages: {:#}", err);
-                            connection.trigger_reconnect(&ctx);
-                        }
-                    }
-                }
-                Err(err) => {
-                    warn!(ctx, "Can't get Config::FetchedExistingMsgs: {:#}", err);
+            match inbox_fetch_idle(&ctx, &mut connection, session).await {
+                Err(err) => warn!(ctx, "Failed fetch_idle: {err:#}"),
+                Ok(session) => {
+                    old_session = Some(session);
                 }
             }
-
-            if let Err(err) = download_msgs(&ctx, &mut connection).await {
-                warn!(ctx, "Failed to download messages: {:#}", err);
-            }
-
-            if let Err(err) = connection.fetch_metadata(&ctx).await {
-                warn!(ctx, "Failed to fetch metadata: {err:#}.");
-            }
-
-            fetch_idle(&ctx, &mut connection, FolderMeaning::Inbox).await;
         }
     };
 
@@ -515,89 +457,127 @@ pub async fn convert_folder_meaning(
     Ok((folder_config, watch_folder))
 }
 
-/// Implement a single iteration of IMAP loop.
-///
-/// This function performs all IMAP operations on a single folder, selecting it if necessary and
-/// handling all the errors. In case of an error, it is logged, but not propagated upwards. If
-/// critical operation fails such as fetching new messages fails, connection is reset via
-/// `trigger_reconnect`, so a fresh one can be opened.
-async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: FolderMeaning) {
-    let create_mvbox = true;
-    if let Err(err) = connection
-        .ensure_configured_folders(ctx, create_mvbox)
-        .await
-    {
-        warn!(
-            ctx,
-            "Cannot watch {folder_meaning}, ensure_configured_folders() failed: {:#}", err,
-        );
-        connection
-            .fake_idle(ctx, None, FolderMeaning::Unknown)
-            .await;
-        return;
+async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) -> Result<Session> {
+    // Update quota no more than once a minute.
+    let quota_needs_update = {
+        let quota = ctx.quota.read().await;
+        quota
+            .as_ref()
+            .filter(|quota| time_elapsed(&quota.modified) > Duration::from_secs(60))
+            .is_none()
+    };
+    if quota_needs_update {
+        if let Err(err) = ctx.update_recent_quota(&mut session).await {
+            warn!(ctx, "Failed to update quota: {:#}.", err);
+        }
     }
-    let (folder_config, watch_folder) = match convert_folder_meaning(ctx, folder_meaning).await {
-        Ok(meaning) => meaning,
-        Err(error) => {
-            // Warning instead of error because the folder may not be configured.
-            // For example, this happens if the server does not have Sent folder
-            // but watching Sent folder is enabled.
-            warn!(ctx, "Error converting IMAP Folder name: {:?}", error);
-            connection.connectivity.set_not_configured(ctx).await;
-            connection
-                .fake_idle(ctx, None, FolderMeaning::Unknown)
-                .await;
-            return;
+
+    let resync_requested = ctx.resync_request.swap(false, Ordering::Relaxed);
+    if resync_requested {
+        if let Err(err) = session.resync_folders(ctx).await {
+            warn!(ctx, "Failed to resync folders: {:#}.", err);
+            ctx.resync_request.store(true, Ordering::Relaxed);
+        }
+    }
+
+    maybe_add_time_based_warnings(ctx).await;
+
+    match ctx.get_config_i64(Config::LastHousekeeping).await {
+        Ok(last_housekeeping_time) => {
+            let next_housekeeping_time = last_housekeeping_time.saturating_add(60 * 60 * 24);
+            if next_housekeeping_time <= time() {
+                sql::housekeeping(ctx).await.log_err(ctx).ok();
+            }
+        }
+        Err(err) => {
+            warn!(ctx, "Failed to get last housekeeping time: {}", err);
         }
     };
 
-    // connect and fake idle if unable to connect
-    if let Err(err) = connection
-        .prepare(ctx)
-        .await
-        .context("prepare IMAP connection")
-    {
-        warn!(ctx, "{:#}", err);
-        connection.trigger_reconnect(ctx);
-        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
-    }
+    match ctx.get_config_bool(Config::FetchedExistingMsgs).await {
+        Ok(fetched_existing_msgs) => {
+            if !fetched_existing_msgs {
+                // Consider it done even if we fail.
+                //
+                // This operation is not critical enough to retry,
+                // especially if the error is persistent.
+                if let Err(err) = ctx
+                    .set_config_internal(Config::FetchedExistingMsgs, config::from_bool(true))
+                    .await
+                {
+                    warn!(ctx, "Can't set Config::FetchedExistingMsgs: {:#}", err);
+                }
 
-    if folder_config == Config::ConfiguredInboxFolder {
-        if let Some(session) = connection.session.as_mut() {
-            session
-                .store_seen_flags_on_imap(ctx)
-                .await
-                .context("store_seen_flags_on_imap")
-                .log_err(ctx)
-                .ok();
-        } else {
-            warn!(ctx, "No session even though we just prepared it");
+                if let Err(err) = imap.fetch_existing_msgs(ctx, &mut session).await {
+                    warn!(ctx, "Failed to fetch existing messages: {:#}", err);
+                }
+            }
+        }
+        Err(err) => {
+            warn!(ctx, "Can't get Config::FetchedExistingMsgs: {:#}", err);
         }
     }
 
-    // Fetch the watched folder.
-    if let Err(err) = connection
-        .fetch_move_delete(ctx, &watch_folder, folder_meaning)
+    download_msgs(ctx, &mut session)
         .await
-        .context("fetch_move_delete")
-    {
-        connection.trigger_reconnect(ctx);
-        warn!(ctx, "{:#}", err);
-        return;
+        .context("Failed to download messages")?;
+    session
+        .fetch_metadata(ctx)
+        .await
+        .context("Failed to fetch metadata")?;
+    session
+        .register_token(ctx)
+        .await
+        .context("Failed to register push token")?;
+
+    let session = fetch_idle(ctx, imap, session, FolderMeaning::Inbox).await?;
+    Ok(session)
+}
+
+/// Implement a single iteration of IMAP loop.
+///
+/// This function performs all IMAP operations on a single folder, selecting it if necessary and
+/// handling all the errors. In case of an error, an error is returned and connection is dropped,
+/// otherwise connection is returned.
+async fn fetch_idle(
+    ctx: &Context,
+    connection: &mut Imap,
+    mut session: Session,
+    folder_meaning: FolderMeaning,
+) -> Result<Session> {
+    let (folder_config, watch_folder) = match convert_folder_meaning(ctx, folder_meaning).await {
+        Ok(meaning) => meaning,
+        Err(err) => {
+            // Warning instead of error because the folder may not be configured.
+            // For example, this happens if the server does not have Sent folder
+            // but watching Sent folder is enabled.
+            warn!(ctx, "Error converting IMAP Folder name: {err:#}.");
+            connection.connectivity.set_not_configured(ctx).await;
+            connection.idle_interrupt_receiver.recv().await.ok();
+            return Err(err);
+        }
+    };
+
+    if folder_config == Config::ConfiguredInboxFolder {
+        session
+            .store_seen_flags_on_imap(ctx)
+            .await
+            .context("store_seen_flags_on_imap")?;
     }
 
+    // Fetch the watched folder.
+    connection
+        .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
+        .await
+        .context("fetch_move_delete")?;
+
     // Mark expired messages for deletion. Marked messages will be deleted from the server
     // on the next iteration of `fetch_move_delete`. `delete_expired_imap_messages` is not
     // called right before `fetch_move_delete` because it is not well optimized and would
     // otherwise slow down message fetching.
     delete_expired_imap_messages(ctx)
         .await
-        .context("delete_expired_imap_messages")
-        .log_err(ctx)
-        .ok();
+        .context("delete_expired_imap_messages")?;
 
     // Scan additional folders only after finishing fetching the watched folder.
     //
@@ -605,7 +585,11 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
     // be able to scan all folders before time is up if there are many of them.
     if folder_config == Config::ConfiguredInboxFolder {
         // Only scan on the Inbox thread in order to prevent parallel scans, which might lead to duplicate messages
-        match connection.scan_folders(ctx).await.context("scan_folders") {
+        match connection
+            .scan_folders(ctx, &mut session)
+            .await
+            .context("scan_folders")
+        {
             Err(err) => {
                 // Don't reconnect, if there is a problem with the connection we will realize this when IDLEing
                 // but maybe just one folder can't be selected or something
@@ -618,22 +602,17 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
                 // In most cases this will select the watched folder and return because there are
                 // no new messages. We want to select the watched folder anyway before going IDLE
                 // there, so this does not take additional protocol round-trip.
-                if let Err(err) = connection
-                    .fetch_move_delete(ctx, &watch_folder, folder_meaning)
+                connection
+                    .fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
                     .await
-                    .context("fetch_move_delete after scan_folders")
-                {
-                    connection.trigger_reconnect(ctx);
-                    warn!(ctx, "{:#}", err);
-                    return;
-                }
+                    .context("fetch_move_delete after scan_folders")?;
             }
             Ok(false) => {}
         }
     }
 
     // Synchronize Seen flags.
-    connection
+    session
         .sync_seen_flags(ctx, &watch_folder)
         .await
         .context("sync_seen_flags")
@@ -643,13 +622,6 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
     connection.connectivity.set_idle(ctx).await;
 
     ctx.emit_event(EventType::ImapInboxIdle);
-    let Some(session) = connection.session.take() else {
-        warn!(ctx, "No IMAP session, going to fake idle.");
-        connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
-    };
 
     if !session.can_idle() {
         info!(
@@ -657,9 +629,9 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
             "IMAP session does not support IDLE, going to fake idle."
         );
         connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
+            .fake_idle(ctx, &mut session, watch_folder, folder_meaning)
+            .await?;
+        return Ok(session);
     }
 
     if ctx
@@ -671,29 +643,22 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder_meaning: Folder
     {
         info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
         connection
-            .fake_idle(ctx, Some(watch_folder), folder_meaning)
-            .await;
-        return;
+            .fake_idle(ctx, &mut session, watch_folder, folder_meaning)
+            .await?;
+        return Ok(session);
     }
 
     info!(ctx, "IMAP session supports IDLE, using it.");
-    match session
+    let session = session
         .idle(
             ctx,
             connection.idle_interrupt_receiver.clone(),
             &watch_folder,
         )
         .await
-        .context("idle")
-    {
-        Ok(session) => {
-            connection.session = Some(session);
-        }
-        Err(err) => {
-            connection.trigger_reconnect(ctx);
-            warn!(ctx, "{:#}", err);
-        }
-    }
+        .context("idle")?;
+
+    Ok(session)
 }
 
 async fn simple_imap_loop(
@@ -719,8 +684,26 @@ async fn simple_imap_loop(
             return;
         }
 
+        let mut old_session: Option<Session> = None;
         loop {
-            fetch_idle(&ctx, &mut connection, folder_meaning).await;
+            let session = if let Some(session) = old_session.take() {
+                session
+            } else {
+                match connection.prepare(&ctx).await {
+                    Err(err) => {
+                        warn!(ctx, "Failed to prepare connection: {:#}.", err);
+                        continue;
+                    }
+                    Ok(session) => session,
+                }
+            };
+
+            match fetch_idle(&ctx, &mut connection, session, folder_meaning).await {
+                Err(err) => warn!(ctx, "Failed fetch_idle: {err:#}"),
+                Ok(session) => {
+                    old_session = Some(session);
+                }
+            }
         }
     };
 

src/securejoin.rs

@@ -770,10 +770,20 @@ mod tests {
     use crate::stock_str::chat_protection_enabled;
     use crate::test_utils::get_chat_msg;
     use crate::test_utils::{TestContext, TestContextManager};
-    use crate::tools::EmailAddress;
+    use crate::tools::{EmailAddress, SystemTime};
+    use std::time::Duration;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_setup_contact() {
+        test_setup_contact_ex(false).await
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_setup_contact_protection_timestamp() {
+        test_setup_contact_ex(true).await
+    }
+
+    async fn test_setup_contact_ex(check_protection_timestamp: bool) {
         let mut tcm = TestContextManager::new();
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
@@ -862,6 +872,10 @@ mod tests {
         // Check Bob sent the right message.
         let sent = bob.pop_sent_msg().await;
         let msg = alice.parse_msg(&sent).await;
+        let vc_request_with_auth_ts_sent = msg
+            .get_header(HeaderDef::Date)
+            .and_then(|value| mailparse::dateparse(value).ok())
+            .unwrap();
         assert!(msg.was_encrypted());
         assert_eq!(
             msg.get_header(HeaderDef::SecureJoin).unwrap(),
@@ -885,6 +899,10 @@ mod tests {
             .unwrap();
         assert_eq!(contact_bob.is_verified(&alice.ctx).await.unwrap(), false);
 
+        if check_protection_timestamp {
+            SystemTime::shift(Duration::from_secs(3600));
+        }
+
         // Step 5+6: Alice receives vc-request-with-auth, sends vc-contact-confirm
         alice.recv_msg(&sent).await;
         assert_eq!(contact_bob.is_verified(&alice.ctx).await.unwrap(), true);
@@ -910,6 +928,9 @@ mod tests {
             assert!(msg.is_info());
             let expected_text = chat_protection_enabled(&alice).await;
             assert_eq!(msg.get_text(), expected_text);
+            if check_protection_timestamp {
+                assert_eq!(msg.timestamp_sort, vc_request_with_auth_ts_sent);
+            }
         }
 
         // Check Alice sent the right message to Bob.

src/smtp.rs

@@ -804,7 +804,7 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
         .sql
         .execute("DELETE FROM smtp_mdns WHERE retries > 6", [])
         .await?;
-    let msg_row = match context
+    let Some(msg_row) = context
         .sql
         .query_row_optional(
             "SELECT msg_id, from_id FROM smtp_mdns ORDER BY retries LIMIT 1",
@@ -816,9 +816,8 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
             },
         )
         .await?
-    {
-        Some(msg_row) => msg_row,
-        None => return Ok(false),
+    else {
+        return Ok(false);
     };
     let (msg_id, contact_id) = msg_row;
 

src/sql.rs

@@ -21,7 +21,7 @@ use crate::message::{Message, MsgId, Viewtype};
 use crate::param::{Param, Params};
 use crate::peerstate::{deduplicate_peerstates, Peerstate};
 use crate::stock_str;
-use crate::tools::{delete_file, time};
+use crate::tools::{delete_file, time, SystemTime};
 
 /// Extension to [`rusqlite::ToSql`] trait
 /// which also includes [`Send`] and [`Sync`].
@@ -850,9 +850,9 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> {
             Ok(mut dir_handle) => {
                 /* avoid deletion of files that are just created to build a message object */
                 let diff = std::time::Duration::from_secs(60 * 60);
-                let keep_files_newer_than = std::time::SystemTime::now()
+                let keep_files_newer_than = SystemTime::now()
                     .checked_sub(diff)
-                    .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
+                    .unwrap_or(SystemTime::UNIX_EPOCH);
 
                 while let Ok(Some(entry)) = dir_handle.next_entry().await {
                     let name_f = entry.file_name();

src/sync.rs

@@ -276,10 +276,7 @@ impl Context {
                     AddQrToken(token) => self.add_qr_token(token).await,
                     DeleteQrToken(token) => self.delete_qr_token(token).await,
                     AlterChat { id, action } => self.sync_alter_chat(id, action).await,
-                    SyncData::Config { key, val } => match key.is_synced() {
-                        true => self.set_config_ex(Sync::Nosync, *key, Some(val)).await,
-                        false => Ok(()),
-                    },
+                    SyncData::Config { key, val } => self.sync_config(key, val).await,
                 },
                 SyncDataOrUnknown::Unknown(data) => {
                     warn!(self, "Ignored unknown sync item: {data}.");
@@ -319,7 +316,7 @@ impl Context {
 
 #[cfg(test)]
 mod tests {
-    use std::time::{Duration, SystemTime};
+    use std::time::Duration;
 
     use anyhow::bail;
 
@@ -329,6 +326,7 @@ mod tests {
     use crate::contact::{Contact, Origin};
     use crate::test_utils::TestContext;
     use crate::token::Namespace;
+    use crate::tools::SystemTime;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_config_sync_msgs() -> Result<()> {

src/tests/verified_chats.rs

@@ -12,6 +12,7 @@ use crate::mimeparser::SystemMessage;
 use crate::receive_imf::receive_imf;
 use crate::stock_str;
 use crate::test_utils::{get_chat_msg, mark_as_verified, TestContext, TestContextManager};
+use crate::tools::SystemTime;
 use crate::{e2ee, message};
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -42,7 +43,7 @@ async fn check_verified_oneonone_chat(broken_by_classical_email: bool) {
             b"Subject: Re: Message from alice\r\n\
           From: <bob@example.net>\r\n\
           To: <alice@example.org>\r\n\
-          Date: Mon, 12 Dec 2022 14:33:39 +0000\r\n\
+          Date: Mon, 12 Dec 3000 14:33:39 +0000\r\n\
           Message-ID: <abcd@example.net>\r\n\
           \r\n\
           Heyho!\r\n",
@@ -51,26 +52,28 @@ async fn check_verified_oneonone_chat(broken_by_classical_email: bool) {
         .await
         .unwrap()
         .unwrap();
+        // Bob's contact is still verified, but the chat isn't marked as protected anymore
+        let contact = alice.add_or_lookup_contact(&bob).await;
+        assert_eq!(contact.is_verified(&alice).await.unwrap(), true);
+        assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
     } else {
         tcm.section("Bob sets up another Delta Chat device");
         let bob2 = TestContext::new().await;
-        enable_verified_oneonone_chats(&[&bob2]).await;
         bob2.set_name("bob2");
         bob2.configure_addr("bob@example.net").await;
 
+        SystemTime::shift(std::time::Duration::from_secs(3600));
         tcm.send_recv(&bob2, &alice, "Using another device now")
             .await;
+        let contact = alice.add_or_lookup_contact(&bob).await;
+        assert_eq!(contact.is_verified(&alice).await.unwrap(), false);
+        assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
     }
 
-    // Bob's contact is still verified, but the chat isn't marked as protected anymore
-    assert_verified(&alice, &bob, ProtectionStatus::ProtectionBroken).await;
-
     tcm.section("Bob sends another message from DC");
+    SystemTime::shift(std::time::Duration::from_secs(3600));
     tcm.send_recv(&bob, &alice, "Using DC again").await;
 
-    let contact = alice.add_or_lookup_contact(&bob).await;
-    assert_eq!(contact.is_verified(&alice.ctx).await.unwrap(), true);
-
     // Bob's chat is marked as verified again
     assert_verified(&alice, &bob, ProtectionStatus::Protected).await;
 }
@@ -792,10 +795,9 @@ async fn test_create_protected_grp_multidev() -> Result<()> {
     );
 
     let sent = alice.send_text(group_id, "Hey").await;
-    // This sleep is necessary to reproduce the bug when the original message is sorted over the
-    // "protection enabled" message so that these messages have different timestamps. The better way
-    // would be to adjust the system time here if we could mock the system clock for the tests.
-    tokio::time::sleep(std::time::Duration::from_millis(2000)).await;
+    // This time shift is necessary to reproduce the bug when the original message is sorted over
+    // the "protection enabled" message so that these messages have different timestamps.
+    SystemTime::shift(std::time::Duration::from_secs(3600));
     let msg = alice1.recv_msg(&sent).await;
     let group1 = Chat::load_from_db(alice1, msg.chat_id).await?;
     assert_eq!(group1.get_type(), Chattype::Group);
@@ -816,8 +818,10 @@ async fn test_create_protected_grp_multidev() -> Result<()> {
 // ============== Helper Functions ==============
 
 async fn assert_verified(this: &TestContext, other: &TestContext, protected: ProtectionStatus) {
-    let contact = this.add_or_lookup_contact(other).await;
-    assert_eq!(contact.is_verified(this).await.unwrap(), true);
+    if protected != ProtectionStatus::ProtectionBroken {
+        let contact = this.add_or_lookup_contact(other).await;
+        assert_eq!(contact.is_verified(this).await.unwrap(), true);
+    }
 
     let chat = this.get_chat(other).await;
     let (expect_protected, expect_broken) = match protected {

src/timesmearing.rs

@@ -79,11 +79,11 @@ impl SmearedTimestamp {
 
 #[cfg(test)]
 mod tests {
-    use std::time::SystemTime;
-
     use super::*;
     use crate::test_utils::TestContext;
-    use crate::tools::{create_smeared_timestamp, create_smeared_timestamps, smeared_time, time};
+    use crate::tools::{
+        create_smeared_timestamp, create_smeared_timestamps, smeared_time, time, SystemTime,
+    };
 
     #[test]
     fn test_smeared_timestamp() {

src/tools.rs

@@ -15,12 +15,16 @@ use std::str::from_utf8;
 // while being in deep sleep mode, we use `SystemTime` instead, but add an alias for it to document
 // why `Instant` isn't used in those places. Also this can help to switch to another clock impl if
 // we find any.
+use std::time::Duration;
 pub use std::time::SystemTime as Time;
-use std::time::{Duration, SystemTime};
+#[cfg(not(test))]
+pub use std::time::SystemTime;
 
 use anyhow::{bail, Context as _, Result};
 use base64::Engine as _;
 use chrono::{Local, NaiveDateTime, NaiveTime, TimeZone};
+#[cfg(test)]
+pub use deltachat_time::SystemTimeTools as SystemTime;
 use futures::{StreamExt, TryStreamExt};
 use mailparse::dateparse;
 use mailparse::headers::Headers;

src/webxdc.rs

@@ -51,18 +51,6 @@ const WEBXDC_API_VERSION: u32 = 1;
 pub const WEBXDC_SUFFIX: &str = "xdc";
 const WEBXDC_DEFAULT_ICON: &str = "__webxdc__/default-icon.png";
 
-/// Defines the maximal size in bytes of an .xdc file that can be sent.
-///
-/// We introduce a limit to force developer to create small .xdc
-/// to save user's traffic and disk space for a better ux.
-///
-/// The 100K limit should also let .xdc pass worse-quality auto-download filters
-/// which are usually 160K incl. base64 overhead.
-///
-/// The limit is also an experiment to see how small we can go;
-/// it is planned to raise that limit as needed in subsequent versions.
-const WEBXDC_SENDING_LIMIT: u64 = 655360;
-
 /// Raw information read from manifest.toml
 #[derive(Debug, Deserialize, Default)]
 #[non_exhaustive]
@@ -229,23 +217,13 @@ impl Context {
         Ok(true)
     }
 
-    /// ensure that a file is an acceptable webxdc for sending
-    /// (sending has more strict size limits).
+    /// Ensure that a file is an acceptable webxdc for sending.
     pub(crate) async fn ensure_sendable_webxdc_file(&self, path: &Path) -> Result<()> {
         let filename = path.to_str().unwrap_or_default();
         if !filename.ends_with(WEBXDC_SUFFIX) {
             bail!("{} is not a valid webxdc file", filename);
         }
 
-        let size = tokio::fs::metadata(path).await?.len();
-        if size > WEBXDC_SENDING_LIMIT {
-            bail!(
-                "webxdc {} exceeds acceptable size of {} bytes",
-                path.to_str().unwrap_or_default(),
-                WEBXDC_SENDING_LIMIT
-            );
-        }
-
         let valid = match async_zip::read::fs::ZipFileReader::new(path).await {
             Ok(archive) => {
                 if find_zip_entry(archive.file(), "index.html").is_none() {
@@ -887,14 +865,6 @@ mod tests {
     use crate::test_utils::TestContext;
     use crate::{message, sql};
 
-    #[allow(clippy::assertions_on_constants)]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_webxdc_file_limits() -> Result<()> {
-        assert!(WEBXDC_SENDING_LIMIT >= 32768);
-        assert!(WEBXDC_SENDING_LIMIT < 16777216);
-        Ok(())
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_is_webxdc_file() -> Result<()> {
         let t = TestContext::new().await;