Attempt to fix OpenSSL build under nix

This fixes the following identity-misbinding attack:

It appears that Bob’s messages in the SecureJoin protocol do not properly “bind” to Alice’s public
key or fingerprint. Even though Bob’s messages carry Alice’s public key and address as a gossip in
the protected payload, Alice does not reject the message if the gossiped key is different from her
own key. As a result, Mallory could perform an identity-misbinding attack. If Mallory obtained
Alice’s QR invite code, she could change her own QR code to contain the same tokens as in Alice’s QR
code, and convince Bob to scan the modified QR code, possibly as an insider attacker. Mallory would
forward messages from Bob to Alice and craft appropriate responses for Bob on his own. In the end,
Bob would believe he is talking to Mallory, but Alice would believe she is talking to Bob.

.github/workflows/ci.yml

@@ -24,9 +24,11 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.75.0
+      RUSTUP_TOOLCHAIN: 1.76.0
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Install rustfmt and clippy
         run: rustup toolchain install $RUSTUP_TOOLCHAIN --profile minimal --component rustfmt --component clippy
       - name: Cache rust cargo artifacts
@@ -42,7 +44,9 @@ jobs:
     name: cargo deny
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: EmbarkStudios/cargo-deny-action@v1
         with:
           arguments: --all-features --workspace
@@ -53,7 +57,9 @@ jobs:
     name: Check provider database
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Check provider database
         run: scripts/update-provider-database.sh
 
@@ -63,8 +69,9 @@ jobs:
     env:
       RUSTDOCFLAGS: -Dwarnings
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
       - name: Rustdoc
@@ -76,18 +83,20 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.75.0
+            rust: 1.76.0
           - os: windows-latest
-            rust: 1.75.0
+            rust: 1.76.0
           - os: macos-latest
-            rust: 1.75.0
+            rust: 1.76.0
 
-          # Minimum Supported Rust Version = 1.72.0
+          # Minimum Supported Rust Version = 1.74.0
           - os: ubuntu-latest
-            rust: 1.72.0
+            rust: 1.74.0
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install Rust ${{ matrix.rust }}
         run: rustup toolchain install --profile minimal ${{ matrix.rust }}
@@ -111,7 +120,9 @@ jobs:
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -120,7 +131,7 @@ jobs:
         run: cargo build -p deltachat_ffi --features jsonrpc
 
       - name: Upload C library
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug/libdeltachat.a
@@ -133,7 +144,9 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Cache rust cargo artifacts
         uses: swatinem/rust-cache@v2
@@ -142,7 +155,7 @@ jobs:
         run: cargo build -p deltachat-rpc-server
 
       - name: Upload deltachat-rpc-server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: ${{ matrix.os == 'windows-latest' && 'target/debug/deltachat-rpc-server.exe' || 'target/debug/deltachat-rpc-server' }}
@@ -152,8 +165,9 @@ jobs:
     name: Python lint
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install tox
         run: pip install tox
@@ -193,16 +207,18 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Download libdeltachat.a
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-libdeltachat.a
           path: target/debug
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -243,10 +259,12 @@ jobs:
 
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Install python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
 
@@ -254,7 +272,7 @@ jobs:
         run: pip install tox
 
       - name: Download deltachat-rpc-server
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.os }}-deltachat-rpc-server
           path: target/debug

.github/workflows/deltachat-rpc-server.yml

@@ -21,22 +21,27 @@ jobs:
   # Build a version statically linked against musl libc
   # to avoid problems with glibc version incompatibility.
   build_linux:
-    name: Cross-compile deltachat-rpc-server for x86_64, i686, aarch64 and armv7 Linux
-    runs-on: ubuntu-22.04
+    name: Build deltachat-rpc-server for Linux
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [aarch64, armv7l, armv6l, i686, x86_64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install ziglang
-        run: pip install wheel ziglang==0.11.0
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
       - name: Build deltachat-rpc-server binaries
-        run: sh scripts/zig-rpc-server.sh
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}-linux
 
-      - name: Upload dist directory with Linux binaries
-        uses: actions/upload-artifact@v3
+      - name: Upload binary
+        uses: actions/upload-artifact@v4
         with:
-          name: linux
-          path: dist/
+          name: deltachat-rpc-server-${{ matrix.arch }}-linux
+          path: result/bin/deltachat-rpc-server
           if-no-files-found: error
 
   build_windows:
@@ -44,32 +49,23 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - os: windows-latest
-            artifact: win32.exe
-            path: deltachat-rpc-server.exe
-            target: i686-pc-windows-msvc
-
-          - os: windows-latest
-            artifact: win64.exe
-            path: deltachat-rpc-server.exe
-            target: x86_64-pc-windows-msvc
-
-    runs-on: ${{ matrix.os }}
+        arch: [win32, win64]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
 
-      - name: Setup rust target
-        run: rustup target add ${{ matrix.target }}
-
-      - name: Build
-        run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.target }} --features vendored
+      - name: Build deltachat-rpc-server binaries
+        run: nix build .#deltachat-rpc-server-${{ matrix.arch }}
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: deltachat-rpc-server-${{ matrix.artifact }}
-          path: target/${{ matrix.target}}/release/${{ matrix.path }}
+          name: deltachat-rpc-server-${{ matrix.arch }}
+          path: result/bin/deltachat-rpc-server.exe
           if-no-files-found: error
 
   build_macos:
@@ -77,13 +73,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - arch: x86_64
-          - arch: aarch64
+        arch: [x86_64, aarch64]
 
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Setup rust target
         run: rustup target add ${{ matrix.arch }}-apple-darwin
@@ -92,7 +88,7 @@ jobs:
         run: cargo build --release --package deltachat-rpc-server --target ${{ matrix.arch }}-apple-darwin --features vendored
 
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-rpc-server-${{ matrix.arch }}-macos
           path: target/${{ matrix.arch }}-apple-darwin/release/deltachat-rpc-server
@@ -100,53 +96,85 @@ jobs:
 
   publish:
     name: Build wheels and upload binaries to the release
-    needs: ["build_linux", "build_windows", "build_macos"]
+    needs: ["build_linux", "build_macos"]
     permissions:
       contents: write
     runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Download Linux binaries
-        uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
         with:
-          name: linux
-          path: dist/
+          show-progress: false
 
-      - name: Download win32 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux aarch64 binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win32.exe
-          path: deltachat-rpc-server-win32.exe.d
+          name: deltachat-rpc-server-aarch64-linux
+          path: deltachat-rpc-server-aarch64-linux.d
 
-      - name: Download win64 binary
-        uses: actions/download-artifact@v3
+      - name: Download Linux armv7l binary
+        uses: actions/download-artifact@v4
         with:
-          name: deltachat-rpc-server-win64.exe
-          path: deltachat-rpc-server-win64.exe.d
+          name: deltachat-rpc-server-armv7l-linux
+          path: deltachat-rpc-server-armv7l-linux.d
+
+      - name: Download Linux armv6l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv6l-linux
+          path: deltachat-rpc-server-armv6l-linux.d
+
+      - name: Download Linux i686 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-i686-linux
+          path: deltachat-rpc-server-i686-linux.d
+
+      - name: Download Linux x86_64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-linux
+          path: deltachat-rpc-server-x86_64-linux.d
+
+      - name: Download Win32 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win32
+          path: deltachat-rpc-server-win32.d
+
+      - name: Download Win64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win64
+          path: deltachat-rpc-server-win64.d
 
       - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-x86_64-macos
           path: deltachat-rpc-server-x86_64-macos.d
 
       - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: deltachat-rpc-server-aarch64-macos
           path: deltachat-rpc-server-aarch64-macos.d
 
       - name: Flatten dist/ directory
         run: |
-          mv deltachat-rpc-server-win32.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
-          mv deltachat-rpc-server-win64.exe.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
+          mkdir -p dist
+          mv deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-linux
+          mv deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-armv7l-linux
+          mv deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-armv6l-linux
+          mv deltachat-rpc-server-i686-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-i686-linux
+          mv deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-linux
+          mv deltachat-rpc-server-win32.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win32.exe
+          mv deltachat-rpc-server-win64.d/deltachat-rpc-server.exe dist/deltachat-rpc-server-win64.exe
           mv deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-macos
           mv deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-macos
 
       # Python 3.11 is needed for tomllib used in scripts/wheel-rpc-server.py
       - name: Install python 3.12
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.12
 
@@ -160,6 +188,7 @@ jobs:
         run: ls -l dist/
 
       - name: Upload binaries to the GitHub release
+        if: github.event_name == 'release'
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: |

.github/workflows/jsonrpc-client-npm-package.yml

@@ -13,8 +13,9 @@ jobs:
     steps:
       - name: Install tree
         run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -49,7 +50,7 @@ jobs:
           ls -lah
           mv $(find deltachat-jsonrpc-client-*) $DELTACHAT_JSONRPC_TAR_GZ
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-jsonrpc-client.tgz
           path: deltachat-jsonrpc/typescript/${{ env.DELTACHAT_JSONRPC_TAR_GZ }}

.github/workflows/jsonrpc.yml

@@ -14,7 +14,9 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3
         with:

.github/workflows/node-docs.yml

@@ -14,7 +14,9 @@ jobs:
   generate:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
 
       - name: Use Node.js 18.x
         uses: actions/setup-node@v3

.github/workflows/node-package.yml

@@ -14,8 +14,9 @@ jobs:
       matrix:
         os: [macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -56,7 +57,7 @@ jobs:
           tar -zcvf "${{ matrix.os }}.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.os }}
           path: node/${{ matrix.os }}.tar.gz
@@ -74,8 +75,9 @@ jobs:
       - name: Change working directory owner
         run: chown root:root .
 
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"
@@ -125,7 +127,7 @@ jobs:
           tar -zcvf "linux.tar.gz" -C prebuilds .
 
       - name: Upload Prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux
           path: node/linux.tar.gz
@@ -137,8 +139,9 @@ jobs:
     steps:
       - name: Install tree
         run: sudo apt install tree
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v2
         with:
           node-version: "18"
@@ -165,23 +168,23 @@ jobs:
           node --version
           echo $DELTACHAT_NODE_TAR_GZ
       - name: Download Linux prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: linux
       - name: Download macOS prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: macos-latest
       - name: Download Windows prebuild
-        uses: actions/download-artifact@v1
+        uses: actions/download-artifact@v4
         with:
           name: windows-latest
       - shell: bash
         run: |
           mkdir node/prebuilds
-          tar -xvzf linux/linux.tar.gz -C node/prebuilds
-          tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
-          tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
+          tar -xvzf linux.tar.gz -C node/prebuilds
+          tar -xvzf macos-latest.tar.gz -C node/prebuilds
+          tar -xvzf windows-latest.tar.gz -C node/prebuilds
           tree node/prebuilds
           rm -rf linux macos-latest windows-latest
       - name: Install dependencies without running scripts
@@ -201,7 +204,7 @@ jobs:
           ls -lah
           mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
       - name: Upload prebuild
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: deltachat-node.tgz
           path: ${{ env.DELTACHAT_NODE_TAR_GZ }}

.github/workflows/node-tests.yml

@@ -23,8 +23,9 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - uses: actions/setup-node@v3
         with:
           node-version: "18"

.github/workflows/repl.yml

@@ -10,15 +10,17 @@ on:
 jobs:
   build_repl:
     name: Build REPL example
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
       - name: Build
-        run: cargo build -p deltachat-repl --features vendored
-
+        run: nix build .#deltachat-repl-win64
       - name: Upload binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: repl.exe
-          path: "target/debug/deltachat-repl.exe"
+          path: "result/bin/deltachat-repl.exe"

.github/workflows/upload-docs.yml

@@ -1,4 +1,4 @@
-name: Build & Deploy Documentation on rs.delta.chat
+name: Build & Deploy Documentation on rs.delta.chat, c.delta.chat, py.delta.chat
 
 on:
   push:
@@ -6,11 +6,13 @@ on:
       - main
 
 jobs:
-  build:
+  build-rs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat --no-deps --document-private-items
@@ -22,3 +24,41 @@ jobs:
           HOST: "delta.chat"
           SOURCE: "target/doc"
           TARGET: "/var/www/html/rs/"
+
+  build-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build Python documentation
+        run: nix build .#python-docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@py.delta.chat:/home/delta/build/master"
+
+  build-c:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - name: Build C documentation
+        run: nix build .#docs
+      - name: Upload to py.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"

.github/workflows/upload-ffi-docs.yml

@@ -14,15 +14,15 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
       - name: Build the documentation with cargo
         run: |
           cargo doc --package deltachat_ffi --no-deps
       - name: Upload to cffi.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/cffi/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc/ "${{ secrets.USERNAME }}@delta.chat:/var/www/html/cffi/"

.github/workflows/upload-python-docs.yml

@@ -1,25 +0,0 @@
-name: Build & Deploy Documentation on py.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Build Python documentation
-        run: scripts/build-python-docs.sh
-      - name: Upload to py.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: delta
-          KEY: ${{ secrets.CODESPEAK_KEY }}
-          HOST: "lists.codespeak.net"
-          SOURCE: "dist/html/"
-          TARGET: "/home/delta/build/master"

CHANGELOG.md

@@ -1,5 +1,240 @@
 # Changelog
 
+## [1.136.0] - 2024-03-04
+
+### Features / Changes
+
+- Recognise Trash folder by name ([#5275](https://github.com/deltachat/deltachat-core-rust/pull/5275)).
+- Send Chat-Group-Avatar as inline base64 ([#5253](https://github.com/deltachat/deltachat-core-rust/pull/5253)).
+- Self-Reporting: Report number of protected/encrypted/unencrypted chats ([#5292](https://github.com/deltachat/deltachat-core-rust/pull/5292)).
+
+### Fixes
+
+- Don't send sync messages on self-{status,avatar} update from self-sent messages ([#5289](https://github.com/deltachat/deltachat-core-rust/pull/5289)).
+- imap: Allow `maybe_network` to interrupt connection ratelimit.
+- imap: Set connectivity to "connecting" only after ratelimit.
+- Remove `Group-ID` from `Message-ID`.
+- Prioritize protected `Message-ID` over `X-Microsoft-Original-Message-ID`.
+
+### API-Changes
+
+- Make `store_self_keypair` private.
+- Add `ContextBuilder.build()` to build Context without opening.
+- `dc_accounts_set_push_device_token` and `dc_get_push_state` APIs for iOS push notifications.
+
+### Build system
+
+- Tag armv6 wheels with tags accepted by PyPI.
+- Unpin OpenSSL.
+- Remove deprecated `unmaintained` field from deny.toml.
+- Do not vendor OpenSSL when cross-compiling ([#5316](https://github.com/deltachat/deltachat-core-rust/pull/5316)).
+- Increase MSRV to 1.74.0.
+
+### CI
+
+- Upgrade setup-python GitHub Action.
+- Update to Rust 1.76 and fix clippy warnings.
+- Build Python docs with Nix.
+- Upload python docs without GH actions.
+- Upload cffi docs without GH actions.
+- Build c.delta.chat docs with nix.
+
+### Other
+
+- refactor: move more methods from Imap into Session.
+- Add deltachat-time to sources.
+
+### Refactor
+
+- Remove Session from Imap structure.
+- Merge ImapConfig into Imap.
+- Get rid of ImapActionResult.
+- Build contexts using ContextBuilder.
+- Do not send `Secure-Join-Group` in `vg-request`.
+
+### Tests
+
+- Fix `test_verified_oneonone_chat_broken_by_device_change()` ([#5280](https://github.com/deltachat/deltachat-core-rust/pull/5280)).
+- `get_protected_chat()`: Use FFIEventTracker instead of `dc_wait_next_msgs()` ([#5207](https://github.com/deltachat/deltachat-core-rust/pull/5207)).
+- Fixup `tests/test_3_offline.py::TestOfflineAccountBasic::test_wrong_db`.
+- Fix pytest compat ([#5317](https://github.com/deltachat/deltachat-core-rust/pull/5317)).
+
+## [1.135.1] - 2024-02-20
+
+### Features / Changes
+
+- Sync self-avatar across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Sync Config::Selfstatus across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Remove webxdc sending limit.
+
+### Fixes
+
+- Never encrypt `{vc,vg}-request` SecureJoin messages.
+- Apply Autocrypt headers if timestamp is unchanged.
+- `Context::get_info`: Report displayname as "displayname" (w/o underscore).
+
+### Tests
+
+- Mock `SystemTime::now()` for the tests.
+- Add a test on protection message sort timestamp ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+### Build system
+
+- Add flake.nix.
+- Add footer template for git-cliff.
+
+### CI
+
+- Update GitHub Actions `actions/upload-artifact`, `actions/download-artifact`, `actions/checkout`.
+- Build deltachat-repl for Windows with nix.
+- Build deltachat-rpc-server with nix.
+- Try to upload deltachat-rpc-server only on release.
+
+### Refactor
+
+- `create_keypair`: Remove unnecessary `map_err`.
+- Return error with a cause when failing to export keys.
+- Rename incorrectly named variables in `create_keypair`.
+
+## [1.135.1] - 2024-02-20
+
+### CI
+
+- Update actions/upload-artifact.
+- Use actions/download-artifact@v4.
+- Replace download-artifact v1 with v4.
+- Update to actions/checkout@v4.
+- Fixup node-package.yml after artifact actions upgrade.
+
+### Features / Changes
+
+- Mock SystemTime::now() for the tests.
+- Remove webxdc sending limit.
+- Sync self-avatar across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- Sync Config::Selfstatus across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+
+### Fixes
+
+- Context::get_info: Report displayname as "displayname" (w/o underscore).
+- Never encrypt {vc,vg}-request.
+
+### Other
+
+- Cleanup changelog ([#5265](https://github.com/deltachat/deltachat-core-rust/pull/5265))
+
+somehow a whole issue sneaked in :).
+
+### Refactor
+
+- create_keypair: Remove unnecessary map_err.
+- Return error with a cause when failing to export keys.
+- Rename incorrectly named variables in create_keypair.
+
+### Tests
+
+- Add a test on protection message sort timestamp ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+## [1.135.0] - 2024-02-13
+
+### Features / Changes
+
+- Add wildcard pattern support to provider database.
+- Add device message about outgoing undecryptable messages ([#5164](https://github.com/deltachat/deltachat-core-rust/pull/5164)).
+- Context::set_config(): Restart IO scheduler if needed ([#5111](https://github.com/deltachat/deltachat-core-rust/pull/5111)).
+- Server_sent_unsolicited_exists(): Log folder name.
+- Cache system time instead of looking at the clock several times in a row.
+- Basic self-reporting ([#5129](https://github.com/deltachat/deltachat-core-rust/pull/5129)).
+
+### Fixes
+
+- Dehtml: Don't just truncate text when trying to decode ([#5223](https://github.com/deltachat/deltachat-core-rust/pull/5223)).
+- Mark the gossip keys from the message as verified, not the ones from the db ([#5247](https://github.com/deltachat/deltachat-core-rust/pull/5247)).
+- Guarantee immediate message deletion if delete_server_after == 0 ([#5201](https://github.com/deltachat/deltachat-core-rust/pull/5201)).
+- Never allow a message timestamp to be a lot in the future ([#5249](https://github.com/deltachat/deltachat-core-rust/pull/5249)).
+- Imap::configure_mvbox: Do select_with_uidvalidity() before return.
+- ImapSession::select_or_create_folder(): Don't fail if folder is created in parallel.
+- Emit ConfigSynced event on the second device.
+- Create mvbox on setting mvbox_move.
+- Use SystemTime instead of Instant everywhere.
+- Restore database rows removed in previous release; this ensures compatibility when adding second device or importing backup and not all devices run the new core ([#5254](https://github.com/deltachat/deltachat-core-rust/pull/5254))
+
+### Miscellaneous Tasks
+
+- cargo: Bump image from 0.24.7 to 0.24.8.
+- cargo: Bump chrono from 0.4.31 to 0.4.33.
+- cargo: Bump futures-lite from 2.1.0 to 2.2.0.
+- cargo: Bump pin-project from 1.1.3 to 1.1.4.
+- cargo: Bump iana-time-zone from yanked 0.1.59 to 0.1.60.
+- cargo: Bump smallvec from 1.11.2 to 1.13.1.
+- cargo: Bump base64 from 0.21.5 to 0.21.7.
+- cargo: Bump regex from 1.10.2 to 1.10.3.
+- cargo: Bump libc from 0.2.151 to 0.2.153.
+- cargo: Bump reqwest from 0.11.23 to 0.11.24.
+- cargo: Bump axum from 0.7.3 to 0.7.4.
+- cargo: Bump uuid from 1.6.1 to 1.7.0.
+- cargo: Bump fast-socks5 from 0.9.2 to 0.9.5.
+- cargo: Bump serde_json from 1.0.111 to 1.0.113.
+- cargo: Bump syn from 2.0.46 to 2.0.48.
+- cargo: Bump serde from 1.0.194 to 1.0.196.
+- cargo: Bump toml from 0.8.8 to 0.8.10.
+- cargo: Update to strum 0.26.
+- Cargo update.
+- scripts: Do not install deltachat-rpc-client twice.
+
+### Other
+
+- Update welcome image, thanks @paulaluap
+- Merge pull request #5243 from deltachat/dependabot/cargo/pin-project-1.1.4
+- Merge pull request #5241 from deltachat/dependabot/cargo/futures-lite-2.2.0
+- Merge pull request #5236 from deltachat/dependabot/cargo/chrono-0.4.33
+- Merge pull request #5235 from deltachat/dependabot/cargo/image-0.24.8
+
+
+### Refactor
+
+- Resultify token::exists.
+
+### Tests
+
+- Delete_server_after="1" should cause immediate message deletion ([#5201](https://github.com/deltachat/deltachat-core-rust/pull/5201)).
+
+## [1.134.0] - 2024-01-31
+
+### API-Changes
+
+- [**breaking**] JSON-RPC: device message api now requires `Option<MessageData>` instead of `String` for the message ([#5211](https://github.com/deltachat/deltachat-core-rust/pull/5211)).
+- CFFI: add `dc_accounts_background_fetch` and event `DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE`.
+- JSON-RPC: add `accounts_background_fetch`.
+
+### Features / Changes
+
+- `Qr::check_qr()`: Accept i.delta.chat invite links ([#5217](https://github.com/deltachat/deltachat-core-rust/pull/5217)).
+- Add support for IMAP METADATA, fetching `/shared/comment` and `/shared/admin` and displaying it in account info.
+
+### Fixes
+
+- Add tolerance for macOS and iOS changing `#` to `%23`.
+- Do not drop unknown report attachments, such as TLS reports.
+- Treat only "Auto-Submitted: auto-generated" messages as bot-sent ([#5213](https://github.com/deltachat/deltachat-core-rust/pull/5213)).
+- `Chat::resend_msgs`: Guarantee strictly increasing time in the `Date` header.
+- Delete resent messages on receiver side ([#5155](https://github.com/deltachat/deltachat-core-rust/pull/5155)).
+- Fix iOS build issue.
+
+### CI
+
+- Add/remove necessary newlines to fix Python lint.
+
+### Tests
+
+- `test_import_export_online_all`: Send the message to the existing address to avoid errors ([#5220](https://github.com/deltachat/deltachat-core-rust/pull/5220)).
+
+## [1.133.2] - 2024-01-24
+
+### Fixes
+
+- Downgrade OpenSSL from 3.2.0 to 3.1.4 ([#5206](https://github.com/deltachat/deltachat-core-rust/issues/5206))
+- No new chats for MDNs with alias ([#5196](https://github.com/deltachat/deltachat-core-rust/issues/5196)) ([#5199](https://github.com/deltachat/deltachat-core-rust/pull/5199)).
+
 ## [1.133.1] - 2024-01-21
 
 ### API-Changes
@@ -3437,3 +3672,8 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1
 [1.133.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.1...v1.133.0
 [1.133.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.0...v1.133.1
+[1.133.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.1...v1.133.2
+[1.134.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.2...v1.134.0
+[1.135.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.134.0...v1.135.0
+[1.135.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.135.0...v1.135.1
+[1.136.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.135.1...v1.136.0

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.133.1"
+version = "1.136.0"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.72"
+rust-version = "1.74"
 
 [profile.dev]
 debug = 0
@@ -31,62 +31,36 @@ codegen-units = 1
 strip = true
 
 [patch.crates-io]
-imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
-"iroh-blake3" = { git = "https://github.com/n0-computer/iroh-blake3.git", branch = "master" }
+openssl-src = { git = "https://github.com/link2xt/openssl-src-rs.git", branch = "link2xt/copy-symlink" }
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+deltachat-time = { path = "./deltachat-time" }
 format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.5", default-features = false, features = [
-  "runtime-tokio",
-] }
-async-native-tls = { version = "0.5", default-features = false, features = [
-  "runtime-tokio",
-] }
-async-smtp = { version = "0.9", default-features = false, features = [
-  "runtime-tokio",
-] }
-async_zip = { version = "0.0.12", default-features = false, features = [
-  "deflate",
-  "fs",
-] }
+async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
+async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
 backtrace = "0.3"
 base64 = "0.21"
-brotli = { version = "3.4", default-features = false, features = ["std"] }
-chrono = { version = "0.4", default-features = false, features = [
-  "clock",
-  "std",
-] }
+brotli = { version = "3.4", default-features=false, features = ["std"] }
+chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 fast-socks5 = "0.9"
 fd-lock = "4"
 futures = "0.3"
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.7", default-features = false, features = [
-  "gif",
-  "jpeg",
-  "ico",
-  "png",
-  "pnm",
-  "webp",
-  "bmp",
-] }
-iroh = { git = "https://github.com/deltachat/iroh", branch = "0.4-update-quic", default-features = false }
-iroh-net = { git = "https://github.com/n0-computer/iroh", branch = "main" }
-iroh-base = { git = "https://github.com/n0-computer/iroh", branch = "main" }
-iroh-gossip = { git = "https://github.com/n0-computer/iroh", branch = "main", features = [
-  "net",
-] }
-quinn = "0.10"
+image = { version = "0.24.9", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -98,16 +72,16 @@ num-traits = "0.2"
 once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
-pgp = { version = "0.10", default-features = false }
+pgp = { version = "0.11", default-features = false }
 pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
 quick-xml = "0.31"
 quoted_printable = "0.5"
 rand = "0.8"
-regex = "1.9"
-reqwest = { version = "0.11.23", features = ["json"] }
-rusqlite = { version = "0.30", features = ["sqlcipher"] }
+regex = "1.10"
+reqwest = { version = "0.11.24", features = ["json"] }
+rusqlite = { version = "0.31", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1"
@@ -115,17 +89,12 @@ serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
 smallvec = "1"
-strum = "0.25"
-strum_macros = "0.25"
+strum = "0.26"
+strum_macros = "0.26"
 tagger = "4.3.4"
-textwrap = "0.16.0"
+textwrap = "0.16.1"
 thiserror = "1"
-tokio = { version = "1", features = [
-  "fs",
-  "rt-multi-thread",
-  "macros",
-  "time",
-] }
+tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
@@ -136,21 +105,15 @@ uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-anyhow = { version = "1", features = [
-  "backtrace",
-] } # Enable `backtrace` feature in tests.
+anyhow = { version = "1", features = ["backtrace"] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 log = "0.4"
 pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
 testdir = "0.9.0"
-tokio = { version = "1", features = [
-  "parking_lot",
-  "rt-multi-thread",
-  "macros",
-] }
+tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
 pretty_assertions = "1.3.0"
 
 [workspace]
@@ -161,6 +124,7 @@ members = [
   "deltachat-rpc-server",
   "deltachat-ratelimit",
   "deltachat-repl",
+  "deltachat-time",
   "format-flowed",
 ]
 
@@ -198,5 +162,5 @@ internals = []
 vendored = [
   "async-native-tls/vendored",
   "rusqlite/bundled-sqlcipher-vendored-openssl",
-  "reqwest/native-tls-vendored",
+  "reqwest/native-tls-vendored"
 ]

RELEASE.md

@@ -9,13 +9,16 @@ For example, to release version 1.116.0 of the core, do the following steps.
 
 3. Update the changelog: `git cliff --unreleased --tag 1.116.0 --prepend CHANGELOG.md` or `git cliff -u -t 1.116.0 -p CHANGELOG.md`.
 
-4. Update the version by running `scripts/set_core_version.py 1.116.0`.
+4. add a link to compare previous with current version to the end of CHANGELOG.md:
+  `[1.116.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.115.2...v1.116.0`
 
-5. Commit the changes as `chore(release): prepare for 1.116.0`.
+5. Update the version by running `scripts/set_core_version.py 1.116.0`.
+
+6. Commit the changes as `chore(release): prepare for 1.116.0`.
    Optionally, use a separate branch like `prep-1.116.0` for this commit and open a PR for review.
 
-6. Tag the release: `git tag -a v1.116.0`.
+7. Tag the release: `git tag -a v1.116.0`.
 
-7. Push the release tag: `git push origin v1.116.0`.
+8. Push the release tag: `git push origin v1.116.0`.
 
-8. Create a GitHub release: `gh release create v1.116.0 -n ''`.
+9. Create a GitHub release: `gh release create v1.116.0 -n ''`.

cliff.toml

@@ -77,3 +77,17 @@ body = """
 """
 # remove the leading and trailing whitespace from the template
 trim = true
+footer = """
+{% for release in releases -%}
+    {% if release.version -%}
+        {% if release.previous.version -%}
+            [{{ release.version | trim_start_matches(pat="v") }}]: \
+                https://github.com/deltachat/deltachat-core-rust\
+                    /compare/{{ release.previous.version }}..{{ release.version }}
+        {% endif -%}
+    {% else -%}
+        [unreleased]: https://github.com/deltachat/deltachat-core-rust\
+            /compare/{{ release.previous.version }}..HEAD
+    {% endif -%}
+{% endfor %}
+"""

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.133.1"
+version = "1.136.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -423,19 +423,16 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    Sending messages to self is needed for a proper multi-account setup,
  *                    however, on the other hand, may lead to unwanted notifications in non-delta clients.
  * - `sentbox_watch`= 1=watch `Sent`-folder for changes,
- *                    0=do not watch the `Sent`-folder (default),
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
+ *                    0=do not watch the `Sent`-folder (default).
  * - `mvbox_move`   = 1=detect chat messages,
  *                    move them to the `DeltaChat` folder,
  *                    and watch the `DeltaChat` folder for updates (default),
  *                    0=do not move chat-messages
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `only_fetch_mvbox` = 1=Do not fetch messages from folders other than the
  *                    `DeltaChat` folder. Messages will still be fetched from the
  *                    spam folder and `sendbox_watch` will also still be respected
  *                    if enabled.
  *                    0=watch all folders normally (default)
- *                    changes require restarting IO by calling dc_stop_io() and then dc_start_io().
  * - `show_emails`  = DC_SHOW_EMAILS_OFF (0)=
  *                    show direct replies to chats only,
  *                    DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=
@@ -689,6 +686,24 @@ int             dc_get_connectivity          (dc_context_t* context);
 char*           dc_get_connectivity_html     (dc_context_t* context);
 
 
+#define DC_PUSH_NOT_CONNECTED 0
+#define DC_PUSH_HEARTBEAT     1
+#define DC_PUSH_CONNECTED     2
+
+/**
+ * Get the current push notification state.
+ * One of:
+ * - DC_PUSH_NOT_CONNECTED
+ * - DC_PUSH_HEARTBEAT
+ * - DC_PUSH_CONNECTED
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @return Push notification state.
+ */
+int              dc_get_push_state           (dc_context_t* context);
+
+
 /**
  * Standalone version of dc_accounts_all_work_done().
  * Only used by the python tests.
@@ -3151,6 +3166,33 @@ void           dc_accounts_maybe_network        (dc_accounts_t* accounts);
 void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
 
 
+/**
+ * Perform a background fetch for all accounts in parallel with a timeout.
+ * Pauses the scheduler, fetches messages from imap and then resumes the scheduler.
+ *
+ * dc_accounts_background_fetch() was created for the iOS Background fetch.
+ *
+ * The `DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE` event is emitted at the end
+ * even in case of timeout, unless the function fails and returns 0.
+ * Process all events until you get this one and you can safely return to the background
+ * without forgetting to create notifications caused by timing race conditions.
+ *
+ * @memberof dc_accounts_t
+ * @param timeout The timeout in seconds
+ * @return Return 1 if DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE was emitted and 0 otherwise.
+ */
+int            dc_accounts_background_fetch    (dc_accounts_t* accounts, uint64_t timeout);
+
+
+/**
+ * Sets device token for Apple Push Notification service.
+ * Returns immediately.
+ *
+ * @memberof dc_accounts_t
+ * @param token Hexadecimal device token
+ */
+void           dc_accounts_set_push_device_token (dc_accounts_t* accounts, const char *token);
+
 /**
  * Create the event emitter that is used to receive events.
  *
@@ -6255,6 +6297,16 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
 
+/**
+ * Tells that the Background fetch was completed (or timed out).
+ *
+ * This event acts as a marker, when you reach this event you can be sure
+ * that all events emitted during the background fetch were processed.
+ * 
+ * This event is only emitted by the account manager
+ */
+
+#define DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE  2200
 
 /**
  * @}

deltachat-ffi/src/lib.rs

@@ -26,7 +26,7 @@ use anyhow::Context as _;
 use deltachat::chat::{ChatId, ChatVisibility, MessageListOptions, MuteDuration, ProtectionStatus};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::{Contact, ContactId, Origin};
-use deltachat::context::Context;
+use deltachat::context::{Context, ContextBuilder};
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
@@ -34,7 +34,6 @@ use deltachat::message::MsgId;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
-use deltachat::stock_str::StockStrings;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
@@ -104,12 +103,11 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(
-            as_path(dbfile),
-            id,
-            Events::new(),
-            StockStrings::new(),
-        ))
+        block_on(
+            ContextBuilder::new(as_path(dbfile).to_path_buf())
+                .with_id(id)
+                .open(),
+        )
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -133,12 +131,11 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     }
 
     let id = rand::thread_rng().gen();
-    match block_on(Context::new_closed(
-        as_path(dbfile),
-        id,
-        Events::new(),
-        StockStrings::new(),
-    )) {
+    match block_on(
+        ContextBuilder::new(as_path(dbfile).to_path_buf())
+            .with_id(id)
+            .build(),
+    ) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
             eprintln!("failed to create context: {err:#}");
@@ -387,7 +384,7 @@ pub unsafe extern "C" fn dc_get_connectivity(context: *const dc_context_t) -> li
         return 0;
     }
     let ctx = &*context;
-    block_on(async move { ctx.get_connectivity().await as u32 as libc::c_int })
+    block_on(ctx.get_connectivity()) as u32 as libc::c_int
 }
 
 #[no_mangle]
@@ -410,6 +407,16 @@ pub unsafe extern "C" fn dc_get_connectivity_html(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_push_state(context: *const dc_context_t) -> libc::c_int {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_push_state()");
+        return 0;
+    }
+    let ctx = &*context;
+    block_on(ctx.push_state()) as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_all_work_done(context: *mut dc_context_t) -> libc::c_int {
     if context.is_null() {
@@ -559,6 +566,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::ConfigSynced { .. } => 2111,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
+        EventType::AccountsBackgroundFetchDone => 2200,
     }
 }
 
@@ -586,7 +594,8 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::SelfavatarChanged
         | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::ErrorSelfNotInGroup(_) => 0,
+        | EventType::ErrorSelfNotInGroup(_)
+        | EventType::AccountsBackgroundFetchDone => 0,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -646,6 +655,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::SelfavatarChanged
+        | EventType::AccountsBackgroundFetchDone
         | EventType::ConfigSynced { .. } => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
@@ -708,6 +718,7 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
         | EventType::WebxdcInstanceDeleted { .. }
+        | EventType::AccountsBackgroundFetchDone
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -4898,6 +4909,49 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_background_fetch(
+    accounts: *mut dc_accounts_t,
+    timeout_in_seconds: u64,
+) -> libc::c_int {
+    if accounts.is_null() || timeout_in_seconds <= 2 {
+        eprintln!("ignoring careless call to dc_accounts_background_fetch()");
+        return 0;
+    }
+
+    let accounts = &*accounts;
+    block_on(async move {
+        let accounts = accounts.read().await;
+        accounts
+            .background_fetch(Duration::from_secs(timeout_in_seconds))
+            .await;
+    });
+    1
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_accounts_set_push_device_token(
+    accounts: *mut dc_accounts_t,
+    token: *const libc::c_char,
+) {
+    if accounts.is_null() {
+        eprintln!("ignoring careless call to dc_accounts_set_push_device_token()");
+        return;
+    }
+
+    let accounts = &*accounts;
+    let token = to_string_lossy(token);
+
+    block_on(async move {
+        let mut accounts = accounts.write().await;
+        if let Err(err) = accounts.set_push_device_token(&token).await {
+            accounts.emit_event(EventType::Error(format!(
+                "Failed to set notify token: {err:#}."
+            )));
+        }
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     accounts: *mut dc_accounts_t,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.133.1"
+version = "1.136.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -17,7 +17,7 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.9.0"
+tempfile = "3.10.1"
 log = "0.4"
 async-channel = { version = "2.0.0" }
 futures = { version = "0.3.30" }
@@ -26,7 +26,7 @@ yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }
 sanitize-filename = "0.5"
-walkdir = "2.3.3"
+walkdir = "2.5.0"
 base64 = "0.21"
 
 # optional dependencies

deltachat-jsonrpc/src/api.rs

@@ -15,6 +15,7 @@ use deltachat::constants::DC_MSG_ID_DAYMARKER;
 use deltachat::contact::{may_be_valid_addr, Contact, ContactId, Origin};
 use deltachat::context::get_info;
 use deltachat::ephemeral::Timer;
+use deltachat::imex;
 use deltachat::location;
 use deltachat::message::get_msg_read_receipts;
 use deltachat::message::{
@@ -27,7 +28,6 @@ use deltachat::reaction::{get_msg_reactions, send_reaction};
 use deltachat::securejoin;
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
-use deltachat::{imex, webxdc};
 use sanitize_filename::is_sanitized;
 use tokio::fs;
 use tokio::sync::{watch, Mutex, RwLock};
@@ -231,13 +231,27 @@ impl CommandApi {
         Ok(())
     }
 
+    /// Performs a background fetch for all accounts in parallel with a timeout.
+    ///
+    /// The `AccountsBackgroundFetchDone` event is emitted at the end even in case of timeout.
+    /// Process all events until you get this one and you can safely return to the background
+    /// without forgetting to create notifications caused by timing race conditions.
+    async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
+        self.accounts
+            .write()
+            .await
+            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+            .await;
+        Ok(())
+    }
+
     // ---------------------------------------------
     // Methods that work on individual accounts
     // ---------------------------------------------
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -311,6 +325,11 @@ impl CommandApi {
         ctx.get_info().await
     }
 
+    async fn draft_self_report(&self, account_id: u32) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx.draft_self_report().await?.to_u32())
+    }
+
     /// Sets the given configuration key.
     async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
@@ -383,7 +402,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -896,19 +915,35 @@ impl CommandApi {
             .to_u32())
     }
 
-    // for now only text messages, because we only used text messages in desktop thusfar
+    /// Add a message to the device-chat.
+    /// Device-messages usually contain update information
+    /// and some hints that are added during the program runs, multi-device etc.
+    /// The device-message may be defined by a label;
+    /// if a message with the same label was added or skipped before,
+    /// the message is not added again, even if the message was deleted in between.
+    /// If needed, the device-chat is created before.
+    ///
+    /// Sends the `MsgsChanged` event on success.
+    ///
+    /// Setting msg to None will prevent the device message with this label from being added in the future.
     async fn add_device_message(
         &self,
         account_id: u32,
         label: String,
-        text: String,
-    ) -> Result<u32> {
+        msg: Option<MessageData>,
+    ) -> Result<Option<u32>> {
         let ctx = self.get_context(account_id).await?;
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(text);
-        let message_id =
-            deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut msg)).await?;
-        Ok(message_id.to_u32())
+        if let Some(msg) = msg {
+            let mut message = msg.create_message(&ctx).await?;
+            let message_id =
+                deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut message)).await?;
+            if !message_id.is_unset() {
+                return Ok(Some(message_id.to_u32()));
+            }
+        } else {
+            deltachat::chat::add_device_msg(&ctx, Some(&label), None).await?;
+        }
+        Ok(None)
     }
 
     ///  Mark all messages in a chat as _noticed_.
@@ -1675,16 +1710,6 @@ impl CommandApi {
             .await
     }
 
-    async fn join_gossip_topic(
-        &self,
-        account_id: u32,
-        instance_msg_id: u32,
-        topic: String,
-    ) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
-        webxdc::join_gossip_topic(&ctx, MsgId::new(instance_msg_id), &topic).await
-    }
-
     async fn get_webxdc_status_updates(
         &self,
         account_id: u32,
@@ -1818,38 +1843,7 @@ impl CommandApi {
 
     async fn send_msg(&self, account_id: u32, chat_id: u32, data: MessageData) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
-        let mut message = Message::new(if let Some(viewtype) = data.viewtype {
-            viewtype.into()
-        } else if data.file.is_some() {
-            Viewtype::File
-        } else {
-            Viewtype::Text
-        });
-        message.set_text(data.text.unwrap_or_default());
-        if data.html.is_some() {
-            message.set_html(data.html);
-        }
-        if data.override_sender_name.is_some() {
-            message.set_override_sender_name(data.override_sender_name);
-        }
-        if let Some(file) = data.file {
-            message.set_file(file, None);
-        }
-        if let Some((latitude, longitude)) = data.location {
-            message.set_location(latitude, longitude);
-        }
-        if let Some(id) = data.quoted_message_id {
-            message
-                .set_quote(
-                    &ctx,
-                    Some(
-                        &Message::load_from_db(&ctx, MsgId::new(id))
-                            .await
-                            .context("message to quote could not be loaded")?,
-                    ),
-                )
-                .await?;
-        }
+        let mut message = data.create_message(&ctx).await?;
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message)
             .await?
             .to_u32();
@@ -2131,13 +2125,6 @@ async fn set_config(
             value,
         )
         .await?;
-
-        match key {
-            "sentbox_watch" | "mvbox_move" | "only_fetch_mvbox" => {
-                ctx.restart_io_if_running().await;
-            }
-            _ => {}
-        }
     }
     Ok(())
 }

deltachat-jsonrpc/src/api/types/events.rs

@@ -245,6 +245,13 @@ pub enum EventType {
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
     WebxdcInstanceDeleted { msg_id: u32 },
+
+    /// Tells that the Background fetch was completed (or timed out).
+    /// This event acts as a marker, when you reach this event you can be sure
+    /// that all events emitted during the background fetch were processed.
+    ///
+    /// This event is only emitted by the account manager
+    AccountsBackgroundFetchDone,
 }
 
 impl From<CoreEventType> for EventType {
@@ -353,6 +360,7 @@ impl From<CoreEventType> for EventType {
             CoreEventType::WebxdcInstanceDeleted { msg_id } => WebxdcInstanceDeleted {
                 msg_id: msg_id.to_u32(),
             },
+            CoreEventType::AccountsBackgroundFetchDone => AccountsBackgroundFetchDone,
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -548,6 +548,44 @@ pub struct MessageData {
     pub quoted_message_id: Option<u32>,
 }
 
+impl MessageData {
+    pub(crate) async fn create_message(self, context: &Context) -> Result<Message> {
+        let mut message = Message::new(if let Some(viewtype) = self.viewtype {
+            viewtype.into()
+        } else if self.file.is_some() {
+            Viewtype::File
+        } else {
+            Viewtype::Text
+        });
+        message.set_text(self.text.unwrap_or_default());
+        if self.html.is_some() {
+            message.set_html(self.html);
+        }
+        if self.override_sender_name.is_some() {
+            message.set_override_sender_name(self.override_sender_name);
+        }
+        if let Some(file) = self.file {
+            message.set_file(file, None);
+        }
+        if let Some((latitude, longitude)) = self.location {
+            message.set_location(latitude, longitude);
+        }
+        if let Some(id) = self.quoted_message_id {
+            message
+                .set_quote(
+                    context,
+                    Some(
+                        &Message::load_from_db(context, MsgId::new(id))
+                            .await
+                            .context("message to quote could not be loaded")?,
+                    ),
+                )
+                .await?;
+        }
+        Ok(message)
+    }
+}
+
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct MessageReadReceipt {

deltachat-jsonrpc/typescript/package.json

@@ -53,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.133.1"
+  "version": "1.136.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.133.1"
+version = "1.136.0"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -9,9 +9,9 @@ ansi_term = "0.12.1"
 anyhow = "1"
 deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
-log = "0.4.20"
+log = "0.4.21"
 pretty_env_logger = "0.5"
-rusqlite = "0.30"
+rusqlite = "0.31"
 rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 

deltachat-repl/src/main.rs

@@ -9,7 +9,6 @@ extern crate deltachat;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
-use std::path::Path;
 use std::process::Command;
 
 use ansi_term::Color;
@@ -20,8 +19,7 @@ use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::qr_code_generator::get_securejoin_qr_svg;
 use deltachat::securejoin::*;
-use deltachat::stock_str::StockStrings;
-use deltachat::{EventType, Events};
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
@@ -312,7 +310,10 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]), 0, Events::new(), StockStrings::new()).await?;
+    let context = ContextBuilder::new(args[1].clone().into())
+        .with_id(1)
+        .open()
+        .await?;
 
     let events = context.get_event_emitter();
     tokio::task::spawn(async move {
@@ -401,7 +402,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    mut ctx: Context,
+    ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -1,4 +1,5 @@
 """Delta Chat JSON-RPC high-level API"""
+
 from ._utils import AttrDict, run_bot_cli, run_client_cli
 from .account import Account
 from .chat import Chat

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -1,4 +1,5 @@
 """Event loop implementations offering high level event handling/hooking."""
+
 import logging
 from typing import (
     TYPE_CHECKING,

deltachat-rpc-client/src/deltachat_rpc_client/events.py

@@ -1,4 +1,5 @@
 """High-level classes for event processing and filtering."""
+
 import re
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Callable, Iterable, Iterator, Optional, Set, Tuple, Union

deltachat-rpc-client/tests/test_securejoin.py

@@ -164,6 +164,21 @@ def test_qr_readreceipt(acfactory) -> None:
     assert not bob.get_chat_by_contact(bob_contact_charlie)
 
 
+def test_setup_contact_resetup(acfactory) -> None:
+    """Tests that setup contact works after Alice resets the device and changes the key."""
+    alice, bob = acfactory.get_online_accounts(2)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    alice = acfactory.resetup_account(alice)
+
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+
 def test_verified_group_recovery(acfactory) -> None:
     """Tests verified group recovery by reverifying a member and sending a message in a group."""
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.133.1"
+version = "1.136.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -15,7 +15,7 @@ deltachat = { path = "..", default-features = false }
 
 anyhow = "1"
 env_logger = { version = "0.10.0" }
-futures-lite = "2.0.0"
+futures-lite = "2.2.0"
 log = "0.4"
 serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }

deltachat-time/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "deltachat-time"
+version = "1.0.0"
+description = "Time-related tools"
+edition = "2021"
+license = "MPL-2.0"
+
+[dependencies]

deltachat-time/src/lib.rs

@@ -0,0 +1,35 @@
+#![allow(missing_docs)]
+
+use std::sync::RwLock;
+use std::time::{Duration, SystemTime};
+
+static SYSTEM_TIME_SHIFT: RwLock<Duration> = RwLock::new(Duration::new(0, 0));
+
+/// Fake struct for mocking `SystemTime::now()` for test purposes. You still need to use
+/// `SystemTime` as a struct representing a system time.
+pub struct SystemTimeTools();
+
+impl SystemTimeTools {
+    pub const UNIX_EPOCH: SystemTime = SystemTime::UNIX_EPOCH;
+
+    pub fn now() -> SystemTime {
+        return SystemTime::now() + *SYSTEM_TIME_SHIFT.read().unwrap();
+    }
+
+    /// Simulates a system clock forward adjustment by `duration`.
+    pub fn shift(duration: Duration) {
+        *SYSTEM_TIME_SHIFT.write().unwrap() += duration;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        SystemTimeTools::shift(Duration::from_secs(60));
+        let t = SystemTimeTools::now();
+        assert!(t > SystemTime::now());
+    }
+}

deny.toml

@@ -1,5 +1,4 @@
 [advisories]
-unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
@@ -10,6 +9,12 @@ ignore = [
     # There is no fix at the time of writing this (2023-11-28).
     # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
     "RUSTSEC-2023-0071",
+
+    # Unmaintained ansi_term
+    "RUSTSEC-2021-0139",
+
+    # Unmaintained encoding
+    "RUSTSEC-2021-0153",
 ]
 
 [bans]
@@ -18,58 +23,58 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
-    { name = "async-channel", version = "1.9.0" },
-    { name = "base16ct", version = "0.1.1" },
-    { name = "base64", version = "<0.21" },
-    { name = "bitflags", version = "1.3.2" },
-    { name = "block-buffer", version = "<0.10" },
-    { name = "convert_case", version = "0.4.0" },
-    { name = "curve25519-dalek", version = "3.2.0" },
-    { name = "darling_core", version = "<0.14" },
-    { name = "darling_macro", version = "<0.14" },
-    { name = "darling", version = "<0.14" },
-    { name = "der", version = "0.6.1" },
-    { name = "digest", version = "<0.10" },
-    { name = "ed25519-dalek", version = "1.0.1" },
-    { name = "ed25519", version = "1.5.3" },
-    { name = "event-listener", version = "2.5.3" },
-    { name = "fd-lock", version = "3.0.13" },
-    { name = "getrandom", version = "<0.2" },
-    { name = "h2", version = "0.3.22" },
-    { name = "http-body", version = "0.4.5" },
-    { name = "http", version = "0.2.11" },
-    { name = "hyper", version = "0.14.27" },
-    { name = "idna", version = "0.4.0" },
-    { name = "pem-rfc7468", version = "0.6.0" },
-    { name = "pkcs8", version = "0.9.0" },
-    { name = "quick-error", version = "<2.0" },
-    { name = "rand_chacha", version = "<0.3" },
-    { name = "rand_core", version = "<0.6" },
-    { name = "rand", version = "<0.8" },
-    { name = "redox_syscall", version = "0.3.5" },
-    { name = "regex-automata", version = "0.1.10" },
-    { name = "regex-syntax", version = "0.6.29" },
-    { name = "ring", version = "0.16.20" },
-    { name = "sec1", version = "0.3.0" },
-    { name = "sha2", version = "<0.10" },
-    { name = "signature", version = "1.6.4" },
-    { name = "socket2", version = "0.4.9" },
-    { name = "spin", version = "<0.9.6" },
-    { name = "spki", version = "0.6.0" },
-    { name = "syn", version = "1.0.109" },
-    { name = "time", version = "<0.3" },
-    { name = "untrusted", version = "0.7.1" },
-    { name = "wasi", version = "<0.11" },
-    { name = "windows_aarch64_gnullvm", version = "<0.52" },
-    { name = "windows_aarch64_msvc", version = "<0.52" },
-    { name = "windows_i686_gnu", version = "<0.52" },
-    { name = "windows_i686_msvc", version = "<0.52" },
-    { name = "windows-sys", version = "<0.52" },
-    { name = "windows-targets", version = "<0.52" },
-    { name = "windows", version = "0.32.0" },
-    { name = "windows_x86_64_gnullvm", version = "<0.52" },
-    { name = "windows_x86_64_gnu", version = "<0.52" },
-    { name = "windows_x86_64_msvc", version = "<0.52" },
+     { name = "async-channel", version = "1.9.0" },
+     { name = "base16ct", version = "0.1.1" },
+     { name = "base64", version = "<0.21" },
+     { name = "bitflags", version = "1.3.2" },
+     { name = "block-buffer", version = "<0.10" },
+     { name = "convert_case", version = "0.4.0" },
+     { name = "curve25519-dalek", version = "3.2.0" },
+     { name = "darling_core", version = "<0.14" },
+     { name = "darling_macro", version = "<0.14" },
+     { name = "darling", version = "<0.14" },
+     { name = "der", version = "0.6.1" },
+     { name = "digest", version = "<0.10" },
+     { name = "ed25519-dalek", version = "1.0.1" },
+     { name = "ed25519", version = "1.5.3" },
+     { name = "event-listener", version = "2.5.3" },
+     { name = "getrandom", version = "<0.2" },
+     { name = "h2", version = "0.3.22" },
+     { name = "http-body", version = "0.4.5" },
+     { name = "http", version = "0.2.11" },
+     { name = "hyper", version = "0.14.27" },
+     { name = "idna", version = "0.4.0" },
+     { name = "pem-rfc7468", version = "0.6.0" },
+     { name = "pkcs8", version = "0.9.0" },
+     { name = "quick-error", version = "<2.0" },
+     { name = "rand_chacha", version = "<0.3" },
+     { name = "rand_core", version = "<0.6" },
+     { name = "rand", version = "<0.8" },
+     { name = "redox_syscall", version = "0.3.5" },
+     { name = "regex-automata", version = "0.1.10" },
+     { name = "regex-syntax", version = "0.6.29" },
+     { name = "ring", version = "0.16.20" },
+     { name = "sec1", version = "0.3.0" },
+     { name = "sha2", version = "<0.10" },
+     { name = "signature", version = "1.6.4" },
+     { name = "spin", version = "<0.9.6" },
+     { name = "spki", version = "0.6.0" },
+     { name = "syn", version = "1.0.109" },
+     { name = "time", version = "<0.3" },
+     { name = "toml_edit", version = "0.21.1" },
+     { name = "untrusted", version = "0.7.1" },
+     { name = "wasi", version = "<0.11" },
+     { name = "windows_aarch64_gnullvm", version = "<0.52" },
+     { name = "windows_aarch64_msvc", version = "<0.52" },
+     { name = "windows_i686_gnu", version = "<0.52" },
+     { name = "windows_i686_msvc", version = "<0.52" },
+     { name = "windows-sys", version = "<0.52" },
+     { name = "windows-targets", version = "<0.52" },
+     { name = "windows", version = "0.32.0" },
+     { name = "windows_x86_64_gnullvm", version = "<0.52" },
+     { name = "windows_x86_64_gnu", version = "<0.52" },
+     { name = "windows_x86_64_msvc", version = "<0.52" },
+     { name = "winnow", version = "0.5.40" },
 ]
 
 
@@ -79,7 +84,7 @@ allow = [
     "Apache-2.0",
     "BSD-2-Clause",
     "BSD-3-Clause",
-    "BSL-1.0",          # Boost Software License 1.0
+    "BSL-1.0", # Boost Software License 1.0
     "CC0-1.0",
     "ISC",
     "MIT",
@@ -92,13 +97,13 @@ allow = [
 [[licenses.clarify]]
 name = "ring"
 expression = "MIT AND ISC AND OpenSSL"
-license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
+license-files = [
+      { path = "LICENSE", hash = 0xbd0eed23 },
+]
 
 [sources.allow-org]
 # Organisations which we allow git sources from.
 github = [
-    "async-email",
-    "deltachat",
-    "djc",
-    "n0-computer", # iroh
+       "async-email",
+       "deltachat",
 ]

flake.lock

@@ -0,0 +1,163 @@
+{
+  "nodes": {
+    "fenix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1707891749,
+        "narHash": "sha256-SeikNYElHgv8uVMbiA9/pU3Cce7ssIsiM8CnEiwd1Nc=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "3115aab064ef38cccd792c45429af8df43d6d277",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1705309234,
+        "narHash": "sha256-uNRRNRKmJyCRC/8y1RqBkqWBLM034y4qN7EprSdmgyA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ef2e671c3b0c19053962c07dbda38332dcebf26",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "naersk": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1698420672,
+        "narHash": "sha256-/TdeHMPRjjdJub7p7+w55vyABrsJlt5QkznPYy55vKA=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "aeb58d5e8faead8980a807c840232697982d47b9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "naersk",
+        "type": "github"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1705332318,
+        "narHash": "sha256-kcw1yFeJe9N4PjQji9ZeX47jg0p9A0DuU4djKvg1a7I=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3449dc925982ad46246cfc36469baf66e1b64f17",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1707689078,
+        "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1707743206,
+        "narHash": "sha256-AehgH64b28yKobC/DAWYZWkJBxL/vP83vkY+ag2Hhy4=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2d627a2a704708673e56346fcb13d25344b8eaf3",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1707689078,
+        "narHash": "sha256-UUGmRa84ZJHpGZ1WZEBEUOzaPOWG8LZ0yPg1pdDF/yM=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "f9d39fb9aff0efee4a3d5f4a6d7c17701d38a1d8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "fenix": "fenix",
+        "flake-utils": "flake-utils",
+        "naersk": "naersk",
+        "nix-filter": "nix-filter",
+        "nixpkgs": "nixpkgs_3"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1707849817,
+        "narHash": "sha256-If6T0MDErp3/z7DBlpG4bV46IPP+7BWSlgTI88cmbw0=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "a02a219773629686bd8ff123ca1aa995fa50d976",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,331 @@
+{
+  description = "Delta Chat core";
+  inputs = {
+    fenix.url = "github:nix-community/fenix";
+    flake-utils.url = "github:numtide/flake-utils";
+    naersk.url = "github:nix-community/naersk";
+    nix-filter.url = "github:numtide/nix-filter";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+  };
+  outputs = { self, nixpkgs, flake-utils, nix-filter, naersk, fenix }:
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+        fenixPkgs = fenix.packages.${system};
+        naersk' = pkgs.callPackage naersk { };
+        manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
+        cargoLock = {
+          lockFile = ./Cargo.lock;
+          outputHashes = {
+            "email-0.0.20" = "sha256-rV4Uzqt2Qdrfi5Ti1r+Si1c2iW1kKyWLwOgLkQ5JGGw=";
+            "encoded-words-0.2.0" = "sha256-KK9st0hLFh4dsrnLd6D8lC6pRFFs8W+WpZSGMGJcosk=";
+            "lettre-0.9.2" = "sha256-+hU1cFacyyeC9UGVBpS14BWlJjHy90i/3ynMkKAzclk=";
+          };
+        };
+        mkRustPackage = packageName:
+          naersk'.buildPackage {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+          };
+        pkgsWin64 = pkgs.pkgsCross.mingwW64;
+        mkWin64RustPackage = packageName:
+          let
+            rustTarget = "x86_64-pc-windows-gnu";
+          in
+          let
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              pkgsWin64.stdenv.cc
+              pkgsWin64.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsWin64.stdenv.cc}/bin/${pkgsWin64.stdenv.cc.targetPrefix}cc";
+          };
+
+        pkgsWin32 = pkgs.pkgsCross.mingw32;
+        mkWin32RustPackage = packageName:
+          let
+            rustTarget = "i686-pc-windows-gnu";
+          in
+          let
+            toolchainWin = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naerskWin = pkgs.callPackage naersk {
+              cargo = toolchainWin;
+              rustc = toolchainWin;
+            };
+
+            # Get rid of MCF Gthread library.
+            # See <https://github.com/NixOS/nixpkgs/issues/156343>
+            # and <https://discourse.nixos.org/t/statically-linked-mingw-binaries/38395>
+            # for details.
+            #
+            # Use DWARF-2 instead of SJLJ for exception handling.
+            winCC = pkgsWin32.buildPackages.wrapCC (
+              (pkgsWin32.buildPackages.gcc-unwrapped.override
+                ({
+                  threadsCross = {
+                    model = "win32";
+                    package = null;
+                  };
+                })).overrideAttrs (oldAttr: rec{
+                configureFlags = oldAttr.configureFlags ++ [
+                  "--disable-sjlj-exceptions --with-dwarf2"
+                ];
+              })
+            );
+            winStdenv = pkgsWin32.buildPackages.overrideCC pkgsWin32.stdenv winCC;
+          in
+          naerskWin.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            depsBuildBuild = [
+              winCC
+              pkgsWin32.windows.pthreads
+            ];
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${winCC}/bin/${winCC.targetPrefix}cc";
+            LD = "${winCC}/bin/${winCC.targetPrefix}cc";
+          };
+
+        mkCrossRustPackage = rustTarget: crossTarget: packageName:
+          let
+            pkgsCross = import nixpkgs {
+              system = system;
+              crossSystem.config = crossTarget;
+            };
+          in
+          let
+            toolchain = fenixPkgs.combine [
+              fenixPkgs.stable.rustc
+              fenixPkgs.stable.cargo
+              fenixPkgs.targets.${rustTarget}.stable.rust-std
+            ];
+            naersk-lib = pkgs.callPackage naersk {
+              cargo = toolchain;
+              rustc = toolchain;
+            };
+          in
+          naersk-lib.buildPackage rec {
+            pname = packageName;
+            cargoBuildOptions = x: x ++ [ "--package" packageName ];
+            version = manifest.version;
+            strictDeps = true;
+            src = pkgs.lib.cleanSource ./.;
+            auditable = false; # Avoid cargo-auditable failures.
+            doCheck = false; # Disable test as it requires network access.
+
+            nativeBuildInputs = [
+              pkgs.perl # Needed to build vendored OpenSSL.
+            ];
+            CARGO_BUILD_TARGET = rustTarget;
+            TARGET_CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            CARGO_BUILD_RUSTFLAGS = [
+              "-C"
+              "linker=${TARGET_CC}"
+            ];
+
+            CC = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+            LD = "${pkgsCross.stdenv.cc}/bin/${pkgsCross.stdenv.cc.targetPrefix}cc";
+          };
+
+        mk-aarch64-RustPackage = mkCrossRustPackage "aarch64-unknown-linux-musl" "aarch64-unknown-linux-musl";
+        mk-i686-RustPackage = mkCrossRustPackage "i686-unknown-linux-musl" "i686-unknown-linux-musl";
+        mk-x86_64-RustPackage = mkCrossRustPackage "x86_64-unknown-linux-musl" "x86_64-unknown-linux-musl";
+        mk-armv7l-RustPackage = mkCrossRustPackage "armv7-unknown-linux-musleabihf" "armv7l-unknown-linux-musleabihf";
+        mk-armv6l-RustPackage = mkCrossRustPackage "arm-unknown-linux-musleabihf" "armv6l-unknown-linux-musleabihf";
+      in
+      {
+        formatter = pkgs.nixpkgs-fmt;
+
+        packages = rec {
+          # Run with `nix run .#deltachat-repl foo.db`.
+          deltachat-repl = mkRustPackage "deltachat-repl";
+          deltachat-rpc-server = mkRustPackage "deltachat-rpc-server";
+
+          deltachat-repl-win64 = mkWin64RustPackage "deltachat-repl";
+          deltachat-rpc-server-win64 = mkWin64RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-win32 = mkWin32RustPackage "deltachat-repl";
+          deltachat-rpc-server-win32 = mkWin32RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-aarch64-linux = mk-aarch64-RustPackage "deltachat-repl";
+          deltachat-rpc-server-aarch64-linux = mk-aarch64-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-i686-linux = mk-i686-RustPackage "deltachat-repl";
+          deltachat-rpc-server-i686-linux = mk-i686-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-x86_64-linux = mk-x86_64-RustPackage "deltachat-repl";
+          deltachat-rpc-server-x86_64-linux = mk-x86_64-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-armv7l-linux = mk-armv7l-RustPackage "deltachat-repl";
+          deltachat-rpc-server-armv7l-linux = mk-armv7l-RustPackage "deltachat-rpc-server";
+
+          deltachat-repl-armv6l-linux = mk-armv6l-RustPackage "deltachat-repl";
+          deltachat-rpc-server-armv6l-linux = mk-armv6l-RustPackage "deltachat-rpc-server";
+
+          # Run `nix build .#docs` to get C docs generated in `./result/`.
+          docs =
+            pkgs.stdenv.mkDerivation {
+              pname = "docs";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./.;
+              nativeBuildInputs = [ pkgs.doxygen ];
+              buildPhase = ''scripts/run-doxygen.sh'';
+              installPhase = ''mkdir -p $out; cp -av deltachat-ffi/html deltachat-ffi/xml $out'';
+            };
+
+          libdeltachat =
+            pkgs.stdenv.mkDerivation rec {
+              pname = "libdeltachat";
+              version = manifest.version;
+              src = nix-filter.lib {
+                root = ./.;
+
+                # Include only necessary files
+                # to avoid rebuilds e.g. when README.md or flake.nix changes.
+                include = [
+                  ./benches
+                  ./assets
+                  ./Cargo.lock
+                  ./Cargo.toml
+                  ./CMakeLists.txt
+                  ./CONTRIBUTING.md
+                  ./deltachat_derive
+                  ./deltachat-ffi
+                  ./deltachat-jsonrpc
+                  ./deltachat-ratelimit
+                  ./deltachat-repl
+                  ./deltachat-rpc-client
+                  ./deltachat-time
+                  ./deltachat-rpc-server
+                  ./format-flowed
+                  ./release-date.in
+                  ./src
+                ];
+                exclude = [
+                  (nix-filter.lib.matchExt "nix")
+                  "flake.lock"
+                ];
+              };
+              cargoDeps = pkgs.rustPlatform.importCargoLock cargoLock;
+
+              nativeBuildInputs = [
+                pkgs.perl # Needed to build vendored OpenSSL.
+                pkgs.cmake
+                pkgs.rustPlatform.cargoSetupHook
+                pkgs.cargo
+              ];
+
+              postInstall = ''
+                substituteInPlace $out/include/deltachat.h \
+                  --replace __FILE__ '"${placeholder "out"}/include/deltachat.h"'
+              '';
+            };
+
+          deltachat-rpc-client =
+            pkgs.python3Packages.buildPythonPackage rec {
+              pname = "deltachat-rpc-client";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./deltachat-rpc-client;
+              format = "pyproject";
+              propagatedBuildInputs = [
+                pkgs.python3Packages.setuptools
+                pkgs.python3Packages.setuptools_scm
+              ];
+            };
+
+          deltachat-python =
+            pkgs.python3Packages.buildPythonPackage rec {
+              pname = "deltachat-python";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./python;
+              format = "pyproject";
+              buildInputs = [
+                libdeltachat
+              ];
+              nativeBuildInputs = [
+                pkgs.pkg-config
+              ];
+              propagatedBuildInputs = [
+                pkgs.python3Packages.setuptools
+                pkgs.python3Packages.setuptools_scm
+                pkgs.python3Packages.pkgconfig
+                pkgs.python3Packages.cffi
+                pkgs.python3Packages.imap-tools
+                pkgs.python3Packages.pluggy
+                pkgs.python3Packages.requests
+              ];
+            };
+          python-docs =
+            pkgs.stdenv.mkDerivation {
+              pname = "docs";
+              version = manifest.version;
+              src = pkgs.lib.cleanSource ./.;
+              buildInputs = [
+                deltachat-python
+                deltachat-rpc-client
+                pkgs.python3Packages.breathe
+                pkgs.python3Packages.sphinx_rtd_theme
+              ];
+              nativeBuildInputs = [ pkgs.sphinx ];
+              buildPhase = ''sphinx-build -b html -a python/doc/ dist/html'';
+              installPhase = ''mkdir -p $out; cp -av dist/html $out'';
+            };
+        };
+      }
+    );
+}

node/constants.js

@@ -29,6 +29,7 @@ module.exports = {
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,

node/events.js

@@ -36,5 +36,6 @@ module.exports = {
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
-  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
+  2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE'
 }

node/lib/constants.ts

@@ -29,6 +29,7 @@ export enum C {
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
+  DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
@@ -326,4 +327,5 @@ export const EventId2EventName: { [key: number]: string } = {
   2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
+  2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
 }

node/test/test.mjs

@@ -239,7 +239,7 @@ describe('Basic offline Tests', function () {
       'delete_device_after',
       'delete_server_after',
       'deltachat_core_version',
-      'display_name',
+      'displayname',
       'download_limit',
       'e2ee_enabled',
       'entered_account_settings',
@@ -250,6 +250,7 @@ describe('Basic offline Tests', function () {
       'journal_mode',
       'key_gen_type',
       'last_housekeeping',
+      'last_cant_decrypt_outgoing_msgs',
       'level',
       'mdns_enabled',
       'media_quality',

package.json

@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.133.1"
+  "version": "1.136.0"
 }

python/src/deltachat/account.py

@@ -1,6 +1,5 @@
 """Account class implementation."""
 
-
 import os
 from array import array
 from contextlib import contextmanager
@@ -376,22 +375,6 @@ class Account:
         dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
         return (x for x in iter_array(dc_array, lambda x: Message.from_db(self, x)) if x is not None)
 
-    def _wait_next_message_ids(self) -> List[int]:
-        """Return IDs of all next messages from all chats."""
-        dc_array = ffi.gc(lib.dc_wait_next_msgs(self._dc_context), lib.dc_array_unref)
-        return [lib.dc_array_get_id(dc_array, i) for i in range(lib.dc_array_get_cnt(dc_array))]
-
-    def wait_next_incoming_message(self) -> Message:
-        """Waits until the next incoming message
-        with ID higher than given is received and returns it."""
-        while True:
-            message_ids = self._wait_next_message_ids()
-            for msg_id in message_ids:
-                message = Message.from_db(self, msg_id)
-                if message and not message.is_from_self() and not message.is_from_device():
-                    self.set_config("last_msg_id", str(msg_id))
-                    return message
-
     def create_chat(self, obj) -> Chat:
         """Create a 1:1 chat with Account, Contact or e-mail address."""
         return self.create_contact(obj).create_chat()
@@ -478,6 +461,16 @@ class Account:
         msg_ids = [msg.id for msg in messages]
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
+    def resend_messages(self, messages: List[Message]) -> None:
+        """Resend list of messages.
+
+        :param messages: list of :class:`deltachat.message.Message` object.
+        :returns: None
+        """
+        msg_ids = [msg.id for msg in messages]
+        if lib.dc_resend_msgs(self._dc_context, msg_ids, len(msg_ids)) != 1:
+            raise ValueError(f"could not resend messages {msg_ids}")
+
     def delete_messages(self, messages: List[Message]) -> None:
         """delete messages (local and remote).
 

python/src/deltachat/events.py

@@ -182,6 +182,12 @@ class FFIEventTracker:
                 print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)
                 break
 
+    def wait_securejoin_joiner_progress(self, target):
+        while True:
+            event = self.get_matching("DC_EVENT_SECUREJOIN_JOINER_PROGRESS")
+            if event.data2 >= target:
+                break
+
     def wait_idle_inbox_ready(self):
         """Has to be called after start_io() to wait for fetch_existing_msgs to run
         so that new messages are not mistaken for old ones:

python/src/deltachat/testplugin.py

@@ -10,7 +10,6 @@ import time
 import weakref
 import random
 from queue import Queue
-from threading import Event
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
@@ -116,7 +115,7 @@ def pytest_configure(config):
     deltachat.register_global_plugin(la)
 
 
-def pytest_report_header(config, startdir):
+def pytest_report_header(config):
     info = get_core_info()
     summary = [
         "Deltachat core={} sqlite={} journal_mode={}".format(
@@ -592,23 +591,16 @@ class ACFactory:
         return ac1.create_chat(ac2)
 
     def get_protected_chat(self, ac1: Account, ac2: Account):
-        class SetupPlugin:
-            def __init__(self) -> None:
-                self.member_added = Event()
-
-            @account_hookimpl
-            def ac_member_added(self, chat: deltachat.Chat, contact, actor, message):
-                self.member_added.set()
-
-        setupplugin = SetupPlugin()
-        ac1.add_account_plugin(setupplugin)
         chat = ac1.create_group_chat("Protected Group", verified=True)
         qr = chat.get_join_qr()
         ac2.qr_join_chat(qr)
-        setupplugin.member_added.wait()
-        msg = ac2.wait_next_incoming_message()
+        ac2._evtracker.wait_securejoin_joiner_progress(1000)
+        ev = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev.data2)
+        assert msg is not None
         assert msg.text == "Messages are guaranteed to be end-to-end encrypted from now on."
-        msg = ac2.wait_next_incoming_message()
+        msg = ac2._evtracker.wait_next_incoming_message()
+        assert msg is not None
         assert "Member Me " in msg.text and " added by " in msg.text
         return chat
 

python/tests/test_1_online.py

@@ -44,21 +44,21 @@ def test_configure_generate_key(acfactory, lp):
     lp.sec("ac1: send unencrypted message to ac2")
     chat.send_text("message1")
     lp.sec("ac2: waiting for message from ac1")
-    msg_in = ac2.wait_next_incoming_message()
+    msg_in = ac2._evtracker.wait_next_incoming_message()
     assert msg_in.text == "message1"
     assert not msg_in.is_encrypted()
 
     lp.sec("ac2: send encrypted message to ac1")
     msg_in.chat.send_text("message2")
     lp.sec("ac1: waiting for message from ac2")
-    msg2_in = ac1.wait_next_incoming_message()
+    msg2_in = ac1._evtracker.wait_next_incoming_message()
     assert msg2_in.text == "message2"
     assert msg2_in.is_encrypted()
 
     lp.sec("ac1: send encrypted message to ac2")
     msg2_in.chat.send_text("message3")
     lp.sec("ac2: waiting for message from ac1")
-    msg3_in = ac2.wait_next_incoming_message()
+    msg3_in = ac2._evtracker.wait_next_incoming_message()
     assert msg3_in.text == "message3"
     assert msg3_in.is_encrypted()
 
@@ -382,6 +382,21 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
     assert msgs_changed_event.data2 == 0
 
 
+def test_enable_mvbox_move(acfactory, lp):
+    (ac1,) = acfactory.get_online_accounts(1)
+
+    lp.sec("ac2: start without mvbox thread")
+    ac2 = acfactory.new_online_configuring_account(mvbox_move=False)
+    acfactory.bring_accounts_online()
+
+    lp.sec("ac2: configuring mvbox")
+    ac2.set_config("mvbox_move", "1")
+
+    lp.sec("ac1: send message and wait for ac2 to receive it")
+    acfactory.get_accepted_chat(ac1, ac2).send_text("message1")
+    assert ac2._evtracker.wait_next_incoming_message().text == "message1"
+
+
 def test_mvbox_sentbox_threads(acfactory, lp):
     lp.sec("ac1: start with mvbox thread")
     ac1 = acfactory.new_online_configuring_account(mvbox_move=True, sentbox_watch=True)
@@ -505,7 +520,7 @@ def test_forward_encrypted_to_unencrypted(acfactory, lp):
     lp.sec("ac1: send encrypted message to ac2")
     txt = "This should be encrypted"
     chat.send_text(txt)
-    msg = ac2.wait_next_incoming_message()
+    msg = ac2._evtracker.wait_next_incoming_message()
     assert msg.text == txt
     assert msg.is_encrypted()
 
@@ -543,6 +558,27 @@ def test_forward_own_message(acfactory, lp):
     assert msg_in.is_forwarded()
 
 
+def test_resend_message(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat1 = ac1.create_chat(ac2)
+
+    lp.sec("ac1: send message to ac2")
+    chat1.send_text("message")
+
+    lp.sec("ac2: receive message")
+    msg_in = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in.text == "message"
+    chat2 = msg_in.chat
+    chat2_msg_cnt = len(chat2.get_messages())
+
+    lp.sec("ac1: resend message")
+    ac1.resend_messages([msg_in])
+
+    lp.sec("ac2: check that message is deleted")
+    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    assert len(chat2.get_messages()) == chat2_msg_cnt
+
+
 def test_long_group_name(acfactory, lp):
     """See bug https://github.com/deltachat/deltachat-core-rust/issues/3650 "Space added before long
     group names after MIME serialization/deserialization".
@@ -1551,10 +1587,11 @@ def test_reactions_for_a_reordering_move(acfactory, lp):
 
 
 def test_import_export_online_all(acfactory, tmp_path, data, lp):
-    (ac1,) = acfactory.get_online_accounts(1)
+    (ac1, some1) = acfactory.get_online_accounts(2)
 
     lp.sec("create some chat content")
-    chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
+    some1_addr = some1.get_config("addr")
+    chat1 = ac1.create_contact(some1_addr, name="some1").create_chat()
     chat1.send_text("msg1")
     assert len(ac1.get_contacts(query="some1")) == 1
 
@@ -1571,7 +1608,7 @@ def test_import_export_online_all(acfactory, tmp_path, data, lp):
         contacts = ac.get_contacts(query="some1")
         assert len(contacts) == 1
         contact2 = contacts[0]
-        assert contact2.addr == "some1@example.org"
+        assert contact2.addr == some1_addr
         chat2 = contact2.create_chat()
         messages = chat2.get_messages()
         assert len(messages) == 3

python/tests/test_3_offline.py

@@ -52,8 +52,8 @@ class TestOfflineAccountBasic:
     def test_wrong_db(self, tmp_path):
         p = tmp_path / "hello.db"
         p.write_text("123")
-        account = Account(str(p))
-        assert not account.is_open()
+        with pytest.raises(ValueError):
+            _account = Account(str(p))
 
     def test_os_name(self, tmp_path):
         p = tmp_path / "hello.db"

release-date.in

@@ -1 +1 @@
-2024-01-21
+2024-03-04

scripts/README.md

@@ -35,12 +35,8 @@ and an own build machine.
 
 - `run_all.sh` builds Python wheels
 
-- `zig-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Zig toolchain statically linked against musl libc.
-
 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.
 
-- `build-python-docs.sh` builds Python documentation into `dist/html/`.
-
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 

scripts/build-python-docs.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export DCC_RS_TARGET=debug
-export DCC_RS_DEV="$PWD"
-cargo build -p deltachat_ffi --features jsonrpc
-
-python3 -m venv venv
-venv/bin/pip install ./python
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/pip install sphinx breathe sphinx_rtd_theme
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/concourse/docs_wheels.yml

@@ -15,55 +15,6 @@ resources:
       tag_filter: "v*"
 
 jobs:
-  - name: doxygen
-    plan:
-      - get: deltachat-core-rust
-        trigger: true
-
-      # Build Doxygen documentation
-      - task: build-doxygen
-        config:
-          inputs:
-            - name: deltachat-core-rust
-          outputs:
-            - name: c-docs
-          image_resource:
-            source:
-              repository: alpine
-            type: registry-image
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache doxygen git
-                cd deltachat-core-rust
-                scripts/run-doxygen.sh
-                cd ..
-                cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
-
-      - task: upload-c-docs
-        config:
-          inputs:
-            - name: c-docs
-          image_resource:
-            type: registry-image
-            source:
-              repository: alpine
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache rsync openssh-client
-                mkdir -p ~/.ssh
-                chmod 700 ~/.ssh
-                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-                chmod 600 ~/.ssh/id_ed25519
-                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
   - name: python-x86_64
     plan:
       - get: deltachat-core-rust

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.72.0
+RUST_VERSION=1.76.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/create-provider-data-rs.py

@@ -220,9 +220,9 @@ if __name__ == "__main__":
 
     process_dir(Path(sys.argv[1]))
 
-    out_all += "pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
+    out_all += "pub(crate) static PROVIDER_DATA: [(&str, &Provider); " + str(len(domains_set)) + "] = [\n";
     out_all += out_domains
-    out_all += "]));\n\n"
+    out_all += "];\n\n"
 
     out_all += "pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| HashMap::from([\n"
     out_all += out_ids

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
+REV=2f3db24107e4802c2df0aa0a40f0e144006c0a9b
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

scripts/wheel-rpc-server.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
-"""Build Python wheels for deltachat-rpc-server.
-Run scripts/zig-rpc-server.sh first."""
+"""Build Python wheels for deltachat-rpc-server."""
 from pathlib import Path
 from wheel.wheelfile import WheelFile
 import tomllib
@@ -157,8 +156,13 @@ def main():
     )
     build_wheel(
         version,
-        "dist/deltachat-rpc-server-armv7-linux",
-        "py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
+        "dist/deltachat-rpc-server-armv7l-linux",
+        "py3-none-linux_armv7l.manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l",
+    )
+    build_wheel(
+        version,
+        "dist/deltachat-rpc-server-armv6l-linux",
+        "py3-none-linux_armv6l",
     )
     build_wheel(
         version,

scripts/zig-cc

@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-# /// pyproject
-# [run]
-# dependencies = [
-#   "ziglang==0.11.0"
-# ]
-# ///
-import os
-import subprocess
-import sys
-
-
-def flag_filter(flag: str) -> bool:
-    # Workaround for <https://github.com/sfackler/rust-openssl/issues/2043>.
-    if flag == "-latomic":
-        return False
-
-    if flag == "-lc":
-        return False
-    if flag == "-Wl,-melf_i386":
-        return False
-    if "self-contained" in flag and "crt" in flag:
-        return False
-    return True
-
-
-def main():
-    args = [flag for flag in sys.argv[1:] if flag_filter(flag)]
-    zig_target = os.environ["ZIG_TARGET"]
-    zig_cpu = os.environ.get("ZIG_CPU")
-    if zig_cpu:
-        zig_cpu_args = ["-mcpu=" + zig_cpu]
-        args = [x for x in args if not x.startswith("-march")]
-    else:
-        zig_cpu_args = []
-
-    # Disable atomics and use locks instead in OpenSSL.
-    # Zig toolchains do not provide atomics.
-    # This is a workaround for <https://github.com/deltachat/deltachat-core-rust/issues/4799>
-    args += ["-DBROKEN_CLANG_ATOMICS"]
-
-    subprocess.run(
-        [
-            sys.executable,
-            "-m",
-            "ziglang",
-            "cc",
-            "-target",
-            zig_target,
-            *zig_cpu_args,
-            *args,
-        ],
-        check=True,
-    )
-
-
-main()

scripts/zig-rpc-server.sh

@@ -1,50 +0,0 @@
-#!/bin/sh
-#
-# Build statically linked deltachat-rpc-server using zig.
-
-set -x
-set -e
-
-unset RUSTFLAGS
-
-# Pin Rust version to avoid uncontrolled changes in the compiler and linker flags.
-export RUSTUP_TOOLCHAIN=1.72.0
-
-rustup target add i686-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_I686_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="x86-linux-musl" \
-cargo build --release --target i686-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-rustup target add armv7-unknown-linux-musleabihf
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_ARMV7_UNKNOWN_LINUX_MUSLEABIHF_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="arm-linux-musleabihf" \
-ZIG_CPU="generic+v7a+vfp3-d32+thumb2-neon" \
-cargo build --release --target armv7-unknown-linux-musleabihf -p deltachat-rpc-server --features vendored
-
-rustup target add x86_64-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="x86_64-linux-musl" \
-cargo build --release --target x86_64-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-rustup target add aarch64-unknown-linux-musl
-CC="$PWD/scripts/zig-cc" \
-TARGET_CC="$PWD/scripts/zig-cc" \
-CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER="$PWD/scripts/zig-cc" \
-LD="$PWD/scripts/zig-cc" \
-ZIG_TARGET="aarch64-linux-musl" \
-cargo build --release --target aarch64-unknown-linux-musl -p deltachat-rpc-server --features vendored
-
-mkdir -p dist
-cp target/x86_64-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-x86_64-linux
-cp target/i686-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-i686-linux
-cp target/aarch64-unknown-linux-musl/release/deltachat-rpc-server dist/deltachat-rpc-server-aarch64-linux
-cp target/armv7-unknown-linux-musleabihf/release/deltachat-rpc-server dist/deltachat-rpc-server-armv7-linux

spec.md

@@ -119,8 +119,9 @@ All group members form the member list.
 To allow different groups with the same members,
 groups are identified by a group-id.
 The group-id MUST be created only from the characters
-`0`-`9`, `A`-`Z`, `a`-`z` `_` and `-`
-and MUST have a length of at least 11 characters.
+`0`-`9`, `A`-`Z`, `a`-`z` `_` and `-`,
+MUST have a length of at least 11 characters
+and no more than 32 characters.
 
 Groups MUST have a group-name.
 The group-name is any non-zero-length UTF-8 string.

src/accounts.rs

@@ -5,6 +5,7 @@ use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
+use futures::future::join_all;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -16,8 +17,9 @@ use tokio::sync::oneshot;
 #[cfg(not(target_os = "ios"))]
 use tokio::time::{sleep, Duration};
 
-use crate::context::Context;
+use crate::context::{Context, ContextBuilder};
 use crate::events::{Event, EventEmitter, EventType, Events};
+use crate::push::PushSubscriber;
 use crate::stock_str::StockStrings;
 
 /// Account manager, that can handle multiple accounts in a single place.
@@ -36,6 +38,9 @@ pub struct Accounts {
     /// This way changing a translation for one context automatically
     /// changes it for all other contexts.
     pub(crate) stockstrings: StockStrings,
+
+    /// Push notification subscriber shared between accounts.
+    push_subscriber: PushSubscriber,
 }
 
 impl Accounts {
@@ -72,8 +77,9 @@ impl Accounts {
             .context("failed to load accounts config")?;
         let events = Events::new();
         let stockstrings = StockStrings::new();
+        let push_subscriber = PushSubscriber::new();
         let accounts = config
-            .load_accounts(&events, &stockstrings, &dir)
+            .load_accounts(&events, &stockstrings, push_subscriber.clone(), &dir)
             .await
             .context("failed to load accounts")?;
 
@@ -83,6 +89,7 @@ impl Accounts {
             accounts,
             events,
             stockstrings,
+            push_subscriber,
         })
     }
 
@@ -119,13 +126,17 @@ impl Accounts {
         let account_config = self.config.new_account().await?;
         let dbfile = account_config.dbfile(&self.dir);
 
-        let ctx = Context::new(
-            &dbfile,
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .with_push_subscriber(self.push_subscriber.clone())
+            .build()
+            .await?;
+        // Try to open without a passphrase,
+        // but do not return an error if account is passphare-protected.
+        ctx.open("".to_string()).await?;
+
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -134,14 +145,15 @@ impl Accounts {
     /// Adds a new closed account.
     pub async fn add_closed_account(&mut self) -> Result<u32> {
         let account_config = self.config.new_account().await?;
+        let dbfile = account_config.dbfile(&self.dir);
 
-        let ctx = Context::new_closed(
-            &account_config.dbfile(&self.dir),
-            account_config.id,
-            self.events.clone(),
-            self.stockstrings.clone(),
-        )
-        .await?;
+        let ctx = ContextBuilder::new(dbfile)
+            .with_id(account_config.id)
+            .with_events(self.events.clone())
+            .with_stock_strings(self.stockstrings.clone())
+            .with_push_subscriber(self.push_subscriber.clone())
+            .build()
+            .await?;
         self.accounts.insert(account_config.id, ctx);
 
         Ok(account_config.id)
@@ -291,6 +303,42 @@ impl Accounts {
         }
     }
 
+    /// Performs a background fetch for all accounts in parallel.
+    ///
+    /// This is an auxiliary function and not part of public API.
+    /// Use [Accounts::background_fetch] instead.
+    async fn background_fetch_without_timeout(&self) {
+        async fn background_fetch_and_log_error(account: Context) {
+            if let Err(error) = account.background_fetch().await {
+                warn!(account, "{error:#}");
+            }
+        }
+
+        join_all(
+            self.accounts
+                .values()
+                .cloned()
+                .map(background_fetch_and_log_error),
+        )
+        .await;
+    }
+
+    /// Performs a background fetch for all accounts in parallel with a timeout.
+    ///
+    /// The `AccountsBackgroundFetchDone` event is emitted at the end,
+    /// process all events until you get this one and you can safely return to the background
+    /// without forgetting to create notifications caused by timing race conditions.
+    pub async fn background_fetch(&self, timeout: std::time::Duration) {
+        if let Err(_err) =
+            tokio::time::timeout(timeout, self.background_fetch_without_timeout()).await
+        {
+            self.emit_event(EventType::Warning(
+                "Background fetch timed out.".to_string(),
+            ));
+        }
+        self.emit_event(EventType::AccountsBackgroundFetchDone);
+    }
+
     /// Emits a single event.
     pub fn emit_event(&self, event: EventType) {
         self.events.emit(Event { id: 0, typ: event })
@@ -300,6 +348,12 @@ impl Accounts {
     pub fn get_event_emitter(&self) -> EventEmitter {
         self.events.get_emitter()
     }
+
+    /// Sets notification token for Apple Push Notification service.
+    pub async fn set_push_device_token(&mut self, token: &str) -> Result<()> {
+        self.push_subscriber.set_device_token(token).await;
+        Ok(())
+    }
 }
 
 /// Configuration file name.
@@ -485,24 +539,24 @@ impl Config {
         &self,
         events: &Events,
         stockstrings: &StockStrings,
+        push_subscriber: PushSubscriber,
         dir: &Path,
     ) -> Result<BTreeMap<u32, Context>> {
         let mut accounts = BTreeMap::new();
 
         for account_config in &self.inner.accounts {
-            let ctx = Context::new(
-                &account_config.dbfile(dir),
-                account_config.id,
-                events.clone(),
-                stockstrings.clone(),
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "failed to create context from file {:?}",
-                    account_config.dbfile(dir)
-                )
-            })?;
+            let dbfile = account_config.dbfile(dir);
+            let ctx = ContextBuilder::new(dbfile.clone())
+                .with_id(account_config.id)
+                .with_events(events.clone())
+                .with_stock_strings(stockstrings.clone())
+                .with_push_subscriber(push_subscriber.clone())
+                .build()
+                .await
+                .with_context(|| format!("failed to create context from file {:?}", &dbfile))?;
+            // Try to open without a passphrase,
+            // but do not return an error if account is passphare-protected.
+            ctx.open("".to_string()).await?;
 
             accounts.insert(account_config.id, ctx);
         }

src/authres.rs

@@ -216,7 +216,7 @@ async fn update_authservid_candidates(
     if old_ids != new_ids {
         let new_config = new_ids.into_iter().collect::<Vec<_>>().join(" ");
         context
-            .set_config(Config::AuthservIdCandidates, Some(&new_config))
+            .set_config_internal(Config::AuthservIdCandidates, Some(&new_config))
             .await?;
         // Updating the authservid candidates may mean that we now consider
         // emails as "failed" which "passed" previously, so we need to

src/blob.rs

@@ -8,6 +8,7 @@ use std::iter::FusedIterator;
 use std::path::{Path, PathBuf};
 
 use anyhow::{format_err, Context as _, Result};
+use base64::Engine as _;
 use futures::StreamExt;
 use image::{DynamicImage, GenericImageView, ImageFormat, ImageOutputFormat};
 use num_traits::FromPrimitive;
@@ -312,6 +313,30 @@ impl<'a> BlobObject<'a> {
         true
     }
 
+    /// Returns path to the stored Base64-decoded blob.
+    ///
+    /// If `data` represents an image of known format, this adds the corresponding extension to
+    /// `suggested_file_stem`.
+    pub(crate) async fn store_from_base64(
+        context: &Context,
+        data: &str,
+        suggested_file_stem: &str,
+    ) -> Result<String> {
+        let buf = base64::engine::general_purpose::STANDARD.decode(data)?;
+        let ext = if let Ok(format) = image::guess_format(&buf) {
+            if let Some(ext) = format.extensions_str().first() {
+                format!(".{ext}")
+            } else {
+                String::new()
+            }
+        } else {
+            String::new()
+        };
+        let blob =
+            BlobObject::create(context, &format!("{suggested_file_stem}{ext}"), &buf).await?;
+        Ok(blob.as_name().to_string())
+    }
+
     pub async fn recode_to_avatar_size(&mut self, context: &Context) -> Result<()> {
         let blob_abs = self.to_abs_path();
 

src/chat.rs

@@ -6,7 +6,7 @@ use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{anyhow, bail, ensure, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
@@ -44,7 +44,7 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     buf_compress, create_id, create_outgoing_rfc724_mid, create_smeared_timestamp,
     create_smeared_timestamps, get_abs_path, gm2local_offset, improve_single_line_input,
-    smeared_time, strip_rtlo_characters, time, IsNoneOrEmpty,
+    smeared_time, strip_rtlo_characters, time, IsNoneOrEmpty, SystemTime,
 };
 use crate::webxdc::WEBXDC_SUFFIX;
 
@@ -783,7 +783,9 @@ impl ChatId {
 
         context.emit_msgs_changed_without_ids();
 
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
         context.scheduler.interrupt_inbox().await;
 
         if chat.is_self_talk() {
@@ -1793,13 +1795,7 @@ impl Chat {
         let mut location_id = 0;
 
         let from = context.get_primary_self_addr().await?;
-        let new_rfc724_mid = {
-            let grpid = match self.typ {
-                Chattype::Group => Some(self.grpid.as_str()),
-                _ => None,
-            };
-            create_outgoing_rfc724_mid(grpid, &from)
-        };
+        let new_rfc724_mid = create_outgoing_rfc724_mid(&from);
 
         if self.typ == Chattype::Single {
             if let Some(id) = context
@@ -2804,12 +2800,14 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         );
     }
 
+    let now = time();
+
     if rendered_msg.is_gossiped {
-        msg.chat_id.set_gossiped_timestamp(context, time()).await?;
+        msg.chat_id.set_gossiped_timestamp(context, now).await?;
     }
 
     if let Some(last_added_location_id) = rendered_msg.last_added_location_id {
-        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, time()).await {
+        if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, now).await {
             error!(context, "Failed to set kml sent_timestamp: {err:#}.");
         }
         if !msg.hidden {
@@ -2828,7 +2826,7 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
     }
 
     if attach_selfavatar {
-        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, time()).await {
+        if let Err(err) = msg.chat_id.set_selfavatar_timestamp(context, now).await {
             error!(context, "Failed to set selfavatar timestamp: {err:#}.");
         }
     }
@@ -3652,7 +3650,7 @@ pub enum MuteDuration {
     Forever,
 
     /// Chat is muted for a limited period of time.
-    Until(SystemTime),
+    Until(std::time::SystemTime),
 }
 
 impl rusqlite::types::ToSql for MuteDuration {
@@ -4076,6 +4074,7 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             chat_id: msg.chat_id,
             msg_id: msg.id,
         });
+        msg.timestamp_sort = create_smeared_timestamp(context);
         if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
             context.scheduler.interrupt_smtp().await;
         }
@@ -4146,7 +4145,7 @@ pub async fn add_device_msg_with_importance(
     if let Some(msg) = msg {
         chat_id = ChatId::get_for_contact(context, ContactId::DEVICE).await?;
 
-        let rfc724_mid = create_outgoing_rfc724_mid(None, "@device");
+        let rfc724_mid = create_outgoing_rfc724_mid("@device");
         prepare_msg_blob(context, msg).await?;
 
         let timestamp_sent = create_smeared_timestamp(context);
@@ -4265,7 +4264,9 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
             (),
         )
         .await?;
-    context.set_config(Config::QuotaExceeding, None).await?;
+    context
+        .set_config_internal(Config::QuotaExceeding, None)
+        .await?;
     Ok(())
 }
 
@@ -4284,7 +4285,7 @@ pub(crate) async fn add_info_msg_with_cmd(
     parent: Option<&Message>,
     from_id: Option<ContactId>,
 ) -> Result<MsgId> {
-    let rfc724_mid = create_outgoing_rfc724_mid(None, "@device");
+    let rfc724_mid = create_outgoing_rfc724_mid("@device");
     let ephemeral_timer = chat_id.get_ephemeral_timer(context).await?;
 
     let mut param = Params::new();
@@ -5925,11 +5926,11 @@ mod tests {
         // Alice has an SMTP-server replacing the `Message-ID:`-header (as done eg. by outlook.com).
         let sent_msg = alice.pop_sent_msg().await;
         let msg = sent_msg.payload();
-        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 2);
-        assert_eq!(msg.match_indices("References: <Gr.").count(), 1);
-        let msg = msg.replace("Message-ID: <Gr.", "Message-ID: <XXX");
-        assert_eq!(msg.match_indices("Message-ID: <Gr.").count(), 0);
-        assert_eq!(msg.match_indices("References: <Gr.").count(), 1);
+        assert_eq!(msg.match_indices("Message-ID: <Mr.").count(), 2);
+        assert_eq!(msg.match_indices("References: <Mr.").count(), 1);
+        let msg = msg.replace("Message-ID: <Mr.", "Message-ID: <XXX");
+        assert_eq!(msg.match_indices("Message-ID: <Mr.").count(), 0);
+        assert_eq!(msg.match_indices("References: <Mr.").count(), 1);
 
         // Bob receives this message, he may detect group by `References:`- or `Chat-Group:`-header
         receive_imf(&bob, msg.as_bytes(), false).await.unwrap();
@@ -5946,7 +5947,7 @@ mod tests {
         send_text_msg(&bob, bob_chat.id, "ho!".to_string()).await?;
         let sent_msg = bob.pop_sent_msg().await;
         let msg = sent_msg.payload();
-        let msg = msg.replace("Message-ID: <Gr.", "Message-ID: <XXX");
+        let msg = msg.replace("Message-ID: <Mr.", "Message-ID: <XXX");
         let msg = msg.replace("Chat-", "XXXX-");
         assert_eq!(msg.match_indices("Chat-").count(), 0);
 
@@ -6502,6 +6503,7 @@ mod tests {
         // Bob receives all messages
         let bob = TestContext::new_bob().await;
         let msg = bob.recv_msg(&sent1).await;
+        let sent1_ts_sent = msg.timestamp_sent;
         assert_eq!(msg.get_text(), "alice->bob");
         assert_eq!(get_chat_contacts(&bob, msg.chat_id).await?.len(), 2);
         assert_eq!(get_chat_msgs(&bob, msg.chat_id).await?.len(), 1);
@@ -6524,6 +6526,7 @@ mod tests {
         assert_eq!(get_chat_msgs(&claire, msg.chat_id).await?.len(), 2);
         let msg_from = Contact::get_by_id(&claire, msg.get_from_id()).await?;
         assert_eq!(msg_from.get_addr(), "alice@example.org");
+        assert!(sent1_ts_sent < msg.timestamp_sent);
 
         Ok(())
     }

src/config.rs

@@ -5,12 +5,14 @@ use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
+use base64::Engine as _;
 use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
-use strum_macros::{AsRefStr, Display, EnumIter, EnumString};
+use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
+use tokio::fs;
 
 use crate::blob::BlobObject;
-use crate::constants::DC_VERSION_STR;
+use crate::constants::{self, DC_VERSION_STR};
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::events::EventType;
@@ -291,6 +293,9 @@ pub enum Config {
     /// Timestamp of the last time housekeeping was run
     LastHousekeeping,
 
+    /// Timestamp of the last `CantDecryptOutgoingMsgs` notification.
+    LastCantDecryptOutgoingMsgs,
+
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
@@ -348,8 +353,9 @@ pub enum Config {
     /// used for signatures, encryption to self and included in `Autocrypt` header.
     KeyId,
 
-    /// Iroh secret key.
-    IrohSecretKey,
+    /// This key is sent to the self_reporting bot so that the bot can recognize the user
+    /// without storing the email address
+    SelfReportingId,
 }
 
 impl Config {
@@ -357,15 +363,31 @@ impl Config {
     ///
     /// This must be checked on both sides so that if there are different client versions, the
     /// synchronisation of a particular option is either done or not done in both directions.
-    /// Moreover, receivers of a config value need to check if a key can be synced because some
-    /// settings (e.g. Avatar path) could otherwise lead to exfiltration of files from a receiver's
+    /// Moreover, receivers of a config value need to check if a key can be synced because if it is
+    /// a file path, it could otherwise lead to exfiltration of files from a receiver's
     /// device if we assume an attacker to have control of a device in a multi-device setting or if
     /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
     /// mustn't be controlled by other devices.
     pub(crate) fn is_synced(&self) -> bool {
+        // We don't restart IO from the synchronisation code, so this is to be on the safe side.
+        if self.needs_io_restart() {
+            return false;
+        }
         matches!(
             self,
-            Self::Displayname | Self::MdnsEnabled | Self::ShowEmails
+            Self::Displayname
+                | Self::MdnsEnabled
+                | Self::ShowEmails
+                | Self::Selfavatar
+                | Self::Selfstatus,
+        )
+    }
+
+    /// Whether the config option needs an IO scheduler restart to take effect.
+    pub(crate) fn needs_io_restart(&self) -> bool {
+        matches!(
+            self,
+            Config::MvboxMove | Config::OnlyFetchMvbox | Config::SentboxWatch
         )
     }
 }
@@ -487,51 +509,25 @@ impl Context {
         }
     }
 
-    /// Set the given config key.
-    /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
-        self.set_config_ex(key.is_synced().into(), key, value).await
+    /// Executes [`SyncData::Config`] item sent by other device.
+    pub(crate) async fn sync_config(&self, key: &Config, value: &str) -> Result<()> {
+        let config_value;
+        let value = match key {
+            Config::Selfavatar if value.is_empty() => None,
+            Config::Selfavatar => {
+                config_value = BlobObject::store_from_base64(self, value, "avatar").await?;
+                Some(config_value.as_str())
+            }
+            _ => Some(value),
+        };
+        match key.is_synced() {
+            true => self.set_config_ex(Nosync, *key, value).await,
+            false => Ok(()),
+        }
     }
 
-    pub(crate) async fn set_config_ex(
-        &self,
-        sync: sync::Sync,
-        key: Config,
-        mut value: Option<&str>,
-    ) -> Result<()> {
-        let better_value;
+    fn check_config(key: Config, value: Option<&str>) -> Result<()> {
         match key {
-            Config::Selfavatar => {
-                self.sql
-                    .execute("UPDATE contacts SET selfavatar_sent=0;", ())
-                    .await?;
-                match value {
-                    Some(value) => {
-                        let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?;
-                        blob.recode_to_avatar_size(self).await?;
-                        self.sql
-                            .set_raw_config(key.as_ref(), Some(blob.as_name()))
-                            .await?;
-                    }
-                    None => {
-                        self.sql.set_raw_config(key.as_ref(), None).await?;
-                    }
-                }
-                self.emit_event(EventType::SelfavatarChanged);
-            }
-            Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(key.as_ref(), value).await;
-                // Interrupt ephemeral loop to delete old messages immediately.
-                self.scheduler.interrupt_ephemeral_task().await;
-                ret?
-            }
-            Config::Displayname => {
-                if let Some(v) = value {
-                    better_value = improve_single_line_input(v);
-                    value = Some(&better_value);
-                }
-                self.sql.set_raw_config(key.as_ref(), value).await?;
-            }
             Config::Socks5Enabled
             | Config::BccSelf
             | Config::E2eeEnabled
@@ -552,6 +548,79 @@ impl Context {
                     matches!(value, None | Some("0") | Some("1")),
                     "Boolean value must be either 0 or 1"
                 );
+            }
+            _ => (),
+        }
+        Ok(())
+    }
+
+    /// Set the given config key and make it effective.
+    /// This may restart the IO scheduler. If `None` is passed as a value the value is cleared and
+    /// set to the default if there is one.
+    pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
+        Self::check_config(key, value)?;
+
+        let _pause = match key.needs_io_restart() {
+            true => self.scheduler.pause(self.clone()).await?,
+            _ => Default::default(),
+        };
+        self.set_config_internal(key, value).await?;
+        Ok(())
+    }
+
+    pub(crate) async fn set_config_internal(&self, key: Config, value: Option<&str>) -> Result<()> {
+        self.set_config_ex(Sync, key, value).await
+    }
+
+    pub(crate) async fn set_config_ex(
+        &self,
+        sync: sync::Sync,
+        key: Config,
+        mut value: Option<&str>,
+    ) -> Result<()> {
+        Self::check_config(key, value)?;
+        let sync = sync == Sync && key.is_synced();
+        let better_value;
+
+        match key {
+            Config::Selfavatar => {
+                self.sql
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", ())
+                    .await?;
+                match value {
+                    Some(path) => {
+                        let mut blob = BlobObject::new_from_path(self, path.as_ref()).await?;
+                        blob.recode_to_avatar_size(self).await?;
+                        self.sql
+                            .set_raw_config(key.as_ref(), Some(blob.as_name()))
+                            .await?;
+                        if sync {
+                            let buf = fs::read(blob.to_abs_path()).await?;
+                            better_value = base64::engine::general_purpose::STANDARD.encode(buf);
+                            value = Some(&better_value);
+                        }
+                    }
+                    None => {
+                        self.sql.set_raw_config(key.as_ref(), None).await?;
+                        if sync {
+                            better_value = String::new();
+                            value = Some(&better_value);
+                        }
+                    }
+                }
+                self.emit_event(EventType::SelfavatarChanged);
+            }
+            Config::DeleteDeviceAfter => {
+                let ret = self.sql.set_raw_config(key.as_ref(), value).await;
+                // Interrupt ephemeral loop to delete old messages immediately.
+                self.scheduler.interrupt_ephemeral_task().await;
+                ret?
+            }
+            Config::Displayname => {
+                if let Some(v) = value {
+                    better_value = improve_single_line_input(v);
+                    value = Some(&better_value);
+                }
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
             Config::Addr => {
@@ -559,15 +628,22 @@ impl Context {
                     .set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
                     .await?;
             }
+            Config::MvboxMove => {
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+                self.sql
+                    .set_raw_config(constants::DC_FOLDERS_CONFIGURED_KEY, None)
+                    .await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
-
-        if sync != Sync {
+        if key.is_synced() {
+            self.emit_event(EventType::ConfigSynced { key });
+        }
+        if !sync {
             return Ok(());
         }
-        self.emit_event(EventType::ConfigSynced { key });
         let Some(val) = value else {
             return Ok(());
         };
@@ -592,8 +668,7 @@ impl Context {
 
     /// Set the given config to a boolean value.
     pub async fn set_config_bool(&self, key: Config, value: bool) -> Result<()> {
-        self.set_config(key, if value { Some("1") } else { Some("0") })
-            .await?;
+        self.set_config(key, from_bool(value)).await?;
         Ok(())
     }
 
@@ -613,6 +688,11 @@ impl Context {
     }
 }
 
+/// Returns a value for use in `Context::set_config_*()` for the given `bool`.
+pub(crate) fn from_bool(val: bool) -> Option<&'static str> {
+    Some(if val { "1" } else { "0" })
+}
+
 // Separate impl block for self address handling
 impl Context {
     /// Determine whether the specified addr maps to the/a self addr.
@@ -639,13 +719,13 @@ impl Context {
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
         secondary_addrs.retain(|a| !addr_cmp(a, primary_new));
-        self.set_config(
+        self.set_config_internal(
             Config::SecondaryAddrs,
             Some(secondary_addrs.join(" ").as_str()),
         )
         .await?;
 
-        self.set_config(Config::ConfiguredAddr, Some(primary_new))
+        self.set_config_internal(Config::ConfiguredAddr, Some(primary_new))
             .await?;
 
         Ok(())
@@ -699,7 +779,7 @@ mod tests {
 
     use super::*;
     use crate::constants;
-    use crate::test_utils::{sync, TestContext};
+    use crate::test_utils::{sync, TestContext, TestContextManager};
 
     #[test]
     fn test_to_string() {
@@ -871,42 +951,9 @@ mod tests {
         // versions.
         alice0.set_config(Config::MdnsEnabled, None).await?;
         assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
-        alice0
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::MdnsEnabled
-                    }
-                )
-            })
-            .await;
         alice0.set_config_bool(Config::MdnsEnabled, false).await?;
-        alice0
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::MdnsEnabled
-                    }
-                )
-            })
-            .await;
         sync(&alice0, &alice1).await;
         assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
-        alice1
-            .evtracker
-            .get_matching(|e| {
-                matches!(
-                    e,
-                    EventType::ConfigSynced {
-                        key: Config::MdnsEnabled
-                    }
-                )
-            })
-            .await;
 
         let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
         alice0
@@ -926,14 +973,150 @@ mod tests {
         assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
 
         // Usual sync scenario.
+        async fn test_config_str(
+            alice0: &TestContext,
+            alice1: &TestContext,
+            key: Config,
+            val: &str,
+        ) -> Result<()> {
+            alice0.set_config(key, Some(val)).await?;
+            sync(alice0, alice1).await;
+            assert_eq!(alice1.get_config(key).await?, Some(val.to_string()));
+            Ok(())
+        }
+        test_config_str(&alice0, &alice1, Config::Displayname, "Alice Sync").await?;
+        test_config_str(&alice0, &alice1, Config::Selfstatus, "My status").await?;
+
+        assert!(alice0.get_config(Config::Selfavatar).await?.is_none());
+        let file = alice0.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        tokio::fs::write(&file, bytes).await?;
+        alice0
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert!(alice1
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".png"))
+            .is_some());
+        alice0.set_config(Config::Selfavatar, None).await?;
+        sync(&alice0, &alice1).await;
+        assert!(alice1.get_config(Config::Selfavatar).await?.is_none());
+
+        Ok(())
+    }
+
+    /// Sync message mustn't be sent if self-{status,avatar} is changed by a self-sent message.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_no_sync_on_self_sent_msg() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice0 = &tcm.alice().await;
+        let alice1 = &tcm.alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let status = "Synced via usual message";
+        alice0.set_config(Config::Selfstatus, Some(status)).await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let status1 = "Synced via sync message";
+        alice1.set_config(Config::Selfstatus, Some(status1)).await?;
+        tcm.send_recv(alice0, alice1, "hi Alice!").await;
+        assert_eq!(
+            alice1.get_config(Config::Selfstatus).await?,
+            Some(status.to_string())
+        );
+        sync(alice1, alice0).await;
+        assert_eq!(
+            alice0.get_config(Config::Selfstatus).await?,
+            Some(status1.to_string())
+        );
+
+        // Need a chat with another contact to send self-avatar.
+        let bob = &tcm.bob().await;
+        let a0b_chat_id = tcm.send_recv_accept(bob, alice0, "hi").await.chat_id;
+        let file = alice0.dir.path().join("avatar.png");
+        let bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        tokio::fs::write(&file, bytes).await?;
+        alice0
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        alice0.pop_sent_msg().await; // Sync message
+        let file = alice1.dir.path().join("avatar.jpg");
+        let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        tokio::fs::write(&file, bytes).await?;
+        alice1
+            .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
+            .await?;
+        let sent_msg = alice0.send_text(a0b_chat_id, "hi").await;
+        alice1.recv_msg(&sent_msg).await;
+        assert!(alice1
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".png"))
+            .is_some());
+        sync(alice1, alice0).await;
+        assert!(alice0
+            .get_config(Config::Selfavatar)
+            .await?
+            .filter(|path| path.ends_with(".jpg"))
+            .is_some());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_event_config_synced() -> Result<()> {
+        let alice0 = TestContext::new_alice().await;
+        let alice1 = TestContext::new_alice().await;
+        for a in [&alice0, &alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
         alice0
             .set_config(Config::Displayname, Some("Alice Sync"))
             .await?;
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
         sync(&alice0, &alice1).await;
         assert_eq!(
             alice1.get_config(Config::Displayname).await?,
             Some("Alice Sync".to_string())
         );
+        alice1
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
+
+        alice0.set_config(Config::Displayname, None).await?;
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::Displayname
+                    }
+                )
+            })
+            .await;
 
         Ok(())
     }

src/configure.rs

@@ -22,10 +22,10 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use server_params::{expand_param_vector, ServerParams};
 use tokio::task;
 
-use crate::config::Config;
+use crate::config::{self, Config};
 use crate::contact::addr_cmp;
 use crate::context::Context;
-use crate::imap::Imap;
+use crate::imap::{session::Session as ImapSession, Imap};
 use crate::log::LogExt;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
@@ -112,12 +112,13 @@ impl Context {
         let mut param = LoginParam::load_candidate_params(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
         let success = configure(self, &mut param).await;
-        self.set_config(Config::NotifyAboutWrongPw, None).await?;
+        self.set_config_internal(Config::NotifyAboutWrongPw, None)
+            .await?;
 
         on_configure_completed(self, param, old_addr).await?;
 
         success?;
-        self.set_config(Config::NotifyAboutWrongPw, Some("1"))
+        self.set_config_internal(Config::NotifyAboutWrongPw, Some("1"))
             .await?;
         Ok(())
     }
@@ -394,7 +395,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     // Configure IMAP
 
-    let mut imap: Option<Imap> = None;
+    let mut imap: Option<(Imap, ImapSession)> = None;
     let imap_servers: Vec<&ServerParams> = servers
         .iter()
         .filter(|params| params.protocol == Protocol::Imap)
@@ -432,7 +433,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             600 + (800 - 600) * (1 + imap_server_index) / imap_servers_count
         );
     }
-    let mut imap = match imap {
+    let (mut imap, mut imap_session) = match imap {
         Some(imap) => imap,
         None => bail!(nicer_configuration_error(ctx, errors).await),
     };
@@ -453,9 +454,11 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     let create_mvbox = ctx.should_watch_mvbox().await?;
 
-    imap.configure_folders(ctx, create_mvbox).await?;
+    imap.configure_folders(ctx, &mut imap_session, create_mvbox)
+        .await?;
 
-    imap.select_with_uidvalidity(ctx, "INBOX")
+    imap_session
+        .select_with_uidvalidity(ctx, "INBOX")
         .await
         .context("could not read INBOX status")?;
 
@@ -473,7 +476,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     // the trailing underscore is correct
     param.save_as_configured_params(ctx).await?;
-    ctx.set_config(Config::ConfiguredTimestamp, Some(&time().to_string()))
+    ctx.set_config_internal(Config::ConfiguredTimestamp, Some(&time().to_string()))
         .await?;
 
     progress!(ctx, 920);
@@ -481,7 +484,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     e2ee::ensure_secret_key_exists(ctx).await?;
     info!(ctx, "key generation completed");
 
-    ctx.set_config_bool(Config::FetchedExistingMsgs, false)
+    ctx.set_config_internal(Config::FetchedExistingMsgs, config::from_bool(false))
         .await?;
     ctx.scheduler.interrupt_inbox().await;
 
@@ -575,7 +578,7 @@ async fn try_imap_one_param(
     socks5_config: &Option<Socks5Config>,
     addr: &str,
     provider_strict_tls: bool,
-) -> Result<Imap, ConfigurationError> {
+) -> Result<(Imap, ImapSession), ConfigurationError> {
     let inf = format!(
         "imap: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
         param.user,
@@ -613,9 +616,9 @@ async fn try_imap_one_param(
                 msg: format!("{err:#}"),
             })
         }
-        Ok(()) => {
+        Ok(session) => {
             info!(context, "success: {}", inf);
-            Ok(imap)
+            Ok((imap, session))
         }
     }
 }

src/constants.rs

@@ -206,6 +206,8 @@ pub(crate) const WORSE_AVATAR_SIZE: u32 = 128;
 pub const BALANCED_IMAGE_SIZE: u32 = 1280;
 pub const WORSE_IMAGE_SIZE: u32 = 640;
 
+// Key for the folder configuration version (see below).
+pub(crate) const DC_FOLDERS_CONFIGURED_KEY: &str = "folders_configured";
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
 
@@ -214,6 +216,9 @@ pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
 // `max_smtp_rcpt_to` in the provider db.
 pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
 
+/// How far the last quota check needs to be in the past to be checked by the background function (in seconds).
+pub(crate) const DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT: u64 = 12 * 60 * 60; // 12 hours
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -6,7 +6,7 @@ use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
-use std::time::{SystemTime, UNIX_EPOCH};
+use std::time::UNIX_EPOCH;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
@@ -36,7 +36,7 @@ use crate::sql::{self, params_iter};
 use crate::sync::{self, Sync::*};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
-    EmailAddress,
+    EmailAddress, SystemTime,
 };
 use crate::{chat, stock_str};
 
@@ -1537,7 +1537,7 @@ WHERE type=? AND id IN (
 /// as profile images can be set only by receiving messages, this should be always the case, however.
 ///
 /// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
-/// this typically happens if we see message with our own profile image, sent from another device.
+/// this typically happens if we see message with our own profile image.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: ContactId,
@@ -1550,7 +1550,7 @@ pub(crate) async fn set_profile_image(
             if contact_id == ContactId::SELF {
                 if was_encrypted {
                     context
-                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .set_config_ex(Nosync, Config::Selfavatar, Some(profile_image))
                         .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar.");
@@ -1563,7 +1563,9 @@ pub(crate) async fn set_profile_image(
         AvatarAction::Delete => {
             if contact_id == ContactId::SELF {
                 if was_encrypted {
-                    context.set_config(Config::Selfavatar, None).await?;
+                    context
+                        .set_config_ex(Nosync, Config::Selfavatar, None)
+                        .await?;
                 } else {
                     info!(context, "Do not use unencrypted selfavatar deletion.");
                 }
@@ -1595,7 +1597,7 @@ pub(crate) async fn set_status(
     if contact_id == ContactId::SELF {
         if encrypted && has_chat_version {
             context
-                .set_config(Config::Selfstatus, Some(&status))
+                .set_config_ex(Nosync, Config::Selfstatus, Some(&status))
                 .await?;
         }
     } else {
@@ -1730,6 +1732,12 @@ impl RecentlySeenLoop {
     async fn run(context: Context, interrupt: Receiver<RecentlySeenInterrupt>) {
         type MyHeapElem = (Reverse<i64>, ContactId);
 
+        let now = SystemTime::now();
+        let now_ts = now
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs() as i64;
+
         // Priority contains all recently seen sorted by the timestamp
         // when they become not recently seen.
         //
@@ -1740,7 +1748,7 @@ impl RecentlySeenLoop {
             .query_map(
                 "SELECT id, last_seen FROM contacts
                  WHERE last_seen > ?",
-                (time() - SEEN_RECENTLY_SECONDS,),
+                (now_ts - SEEN_RECENTLY_SECONDS,),
                 |row| {
                     let contact_id: ContactId = row.get("id")?;
                     let last_seen: i64 = row.get("last_seen")?;
@@ -1755,8 +1763,6 @@ impl RecentlySeenLoop {
             .unwrap_or_default();
 
         loop {
-            let now = SystemTime::now();
-
             let (until, contact_id) =
                 if let Some((Reverse(timestamp), contact_id)) = unseen_queue.peek() {
                     (

src/context.rs

@@ -6,30 +6,37 @@ use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
-use std::time::{Duration, Instant, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
-use iroh_gossip::net::Gossip;
-use iroh_net::MagicEndpoint;
+use pgp::SignedPublicKey;
 use ratelimit::Ratelimit;
 use tokio::sync::{Mutex, Notify, RwLock};
 
-use crate::chat::{get_chat_cnt, ChatId};
+use crate::aheader::EncryptPreference;
+use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus};
 use crate::config::Config;
-use crate::constants::DC_VERSION_STR;
-use crate::contact::Contact;
+use crate::constants::{
+    self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR,
+};
+use crate::contact::{Contact, ContactId};
 use crate::debug_logging::DebugLogging;
+use crate::download::DownloadState;
 use crate::events::{Event, EventEmitter, EventType, Events};
-use crate::key::{load_self_public_key, DcKey as _};
+use crate::imap::{FolderMeaning, Imap, ServerMetadata};
+use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _};
 use crate::login_param::LoginParam;
-use crate::message::{self, MessageState, MsgId};
+use crate::message::{self, Message, MessageState, MsgId, Viewtype};
+use crate::param::{Param, Params};
+use crate::peerstate::Peerstate;
+use crate::push::PushSubscriber;
 use crate::quota::QuotaInfo;
-use crate::scheduler::SchedulerState;
+use crate::scheduler::{convert_folder_meaning, SchedulerState};
 use crate::sql::Sql;
 use crate::stock_str::StockStrings;
 use crate::timesmearing::SmearedTimestamp;
-use crate::tools::{duration_to_str, time};
+use crate::tools::{self, create_id, duration_to_str, time, time_elapsed};
 
 /// Builder for the [`Context`].
 ///
@@ -80,6 +87,8 @@ pub struct ContextBuilder {
     events: Events,
     stock_strings: StockStrings,
     password: Option<String>,
+
+    push_subscriber: Option<PushSubscriber>,
 }
 
 impl ContextBuilder {
@@ -95,6 +104,7 @@ impl ContextBuilder {
             events: Events::new(),
             stock_strings: StockStrings::new(),
             password: None,
+            push_subscriber: None,
         }
     }
 
@@ -149,11 +159,32 @@ impl ContextBuilder {
         self
     }
 
-    /// Opens the [`Context`].
+    /// Sets push subscriber.
+    pub(crate) fn with_push_subscriber(mut self, push_subscriber: PushSubscriber) -> Self {
+        self.push_subscriber = Some(push_subscriber);
+        self
+    }
+
+    /// Builds the [`Context`] without opening it.
+    pub async fn build(self) -> Result<Context> {
+        let push_subscriber = self.push_subscriber.unwrap_or_default();
+        let context = Context::new_closed(
+            &self.dbfile,
+            self.id,
+            self.events,
+            self.stock_strings,
+            push_subscriber,
+        )
+        .await?;
+        Ok(context)
+    }
+
+    /// Builds the [`Context`] and opens it.
+    ///
+    /// Returns error if context cannot be opened with the given passphrase.
     pub async fn open(self) -> Result<Context> {
-        let context =
-            Context::new_closed(&self.dbfile, self.id, self.events, self.stock_strings).await?;
-        let password = self.password.unwrap_or_default();
+        let password = self.password.clone().unwrap_or_default();
+        let context = self.build().await?;
         match context.open(password).await? {
             true => Ok(context),
             false => bail!("database could not be decrypted, incorrect or missing password"),
@@ -226,7 +257,10 @@ pub struct InnerContext {
     /// <https://datatracker.ietf.org/doc/html/rfc2971>
     pub(crate) server_id: RwLock<Option<HashMap<String, String>>>,
 
-    pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
+    /// IMAP METADATA.
+    pub(crate) metadata: RwLock<Option<ServerMetadata>>,
+
+    pub(crate) last_full_folder_scan: Mutex<Option<tools::Time>>,
 
     /// ID for this `Context` in the current process.
     ///
@@ -234,7 +268,7 @@ pub struct InnerContext {
     /// be identified by this ID.
     pub(crate) id: u32,
 
-    creation_time: SystemTime,
+    creation_time: tools::Time,
 
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
@@ -247,11 +281,12 @@ pub struct InnerContext {
     /// because the lock is used from synchronous [`Context::emit_event`].
     pub(crate) debug_logging: std::sync::RwLock<Option<DebugLogging>>,
 
-    /// [MagicEndpoint] needed for iroh peer channels.
-    pub(crate) endpoint: Mutex<Option<MagicEndpoint>>,
+    /// Push subscriber to store device token
+    /// and register for heartbeat notifications.
+    pub(crate) push_subscriber: PushSubscriber,
 
-    /// [Gossip] needed for iroh peer channels.
-    pub(crate) gossip: Mutex<Option<Gossip>>,
+    /// True if account has subscribed to push notifications via IMAP.
+    pub(crate) push_subscribed: AtomicBool,
 }
 
 /// The state of ongoing process.
@@ -261,7 +296,7 @@ enum RunningState {
     Running { cancel_sender: Sender<()> },
 
     /// Cancel signal has been sent, waiting for ongoing process to be freed.
-    ShallStop { request: Instant },
+    ShallStop { request: tools::Time },
 
     /// There is no ongoing process, a new one can be allocated.
     Stopped,
@@ -297,7 +332,8 @@ impl Context {
         events: Events,
         stock_strings: StockStrings,
     ) -> Result<Context> {
-        let context = Self::new_closed(dbfile, id, events, stock_strings).await?;
+        let context =
+            Self::new_closed(dbfile, id, events, stock_strings, Default::default()).await?;
 
         // Open the database if is not encrypted.
         if context.check_passphrase("".to_string()).await? {
@@ -312,6 +348,7 @@ impl Context {
         id: u32,
         events: Events,
         stockstrings: StockStrings,
+        push_subscriber: PushSubscriber,
     ) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -320,7 +357,14 @@ impl Context {
         if !blobdir.exists() {
             tokio::fs::create_dir_all(&blobdir).await?;
         }
-        let context = Context::with_blobdir(dbfile.into(), blobdir, id, events, stockstrings)?;
+        let context = Context::with_blobdir(
+            dbfile.into(),
+            blobdir,
+            id,
+            events,
+            stockstrings,
+            push_subscriber,
+        )?;
         Ok(context)
     }
 
@@ -363,6 +407,7 @@ impl Context {
         id: u32,
         events: Events,
         stockstrings: StockStrings,
+        push_subscriber: PushSubscriber,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir(),
@@ -392,12 +437,13 @@ impl Context {
             resync_request: AtomicBool::new(false),
             new_msgs_notify,
             server_id: RwLock::new(None),
-            creation_time: std::time::SystemTime::now(),
+            metadata: RwLock::new(None),
+            creation_time: tools::Time::now(),
             last_full_folder_scan: Mutex::new(None),
             last_error: std::sync::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
-            endpoint: Mutex::new(None),
-            gossip: Mutex::new(None),
+            push_subscriber,
+            push_subscribed: AtomicBool::new(false),
         };
 
         let ctx = Context {
@@ -408,7 +454,7 @@ impl Context {
     }
 
     /// Starts the IO scheduler.
-    pub async fn start_io(&mut self) {
+    pub async fn start_io(&self) {
         if !self.is_configured().await.unwrap_or_default() {
             warn!(self, "can not start io on a context that is not configured");
             return;
@@ -427,17 +473,11 @@ impl Context {
                 *lock = Ratelimit::new(Duration::new(3, 0), 3.0);
             }
         }
-
-        if let Err(e) = self.create_gossip().await {
-            warn!(self, "{e}");
-        }
         self.scheduler.start(self.clone()).await;
     }
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
-        self.endpoint.lock().await.take();
-        self.gossip.lock().await.take();
         self.scheduler.stop(self).await;
     }
 
@@ -449,12 +489,61 @@ impl Context {
 
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        if let Some(ref mut endpoint) = *self.endpoint.lock().await {
-            endpoint.network_change().await;
-        }
         self.scheduler.maybe_network().await;
     }
 
+    /// Does a background fetch
+    /// pauses the scheduler and does one imap fetch, then unpauses and returns
+    pub async fn background_fetch(&self) -> Result<()> {
+        if !(self.is_configured().await?) {
+            return Ok(());
+        }
+
+        let address = self.get_primary_self_addr().await?;
+        let time_start = tools::Time::now();
+        info!(self, "background_fetch started fetching {address}");
+
+        let _pause_guard = self.scheduler.pause(self.clone()).await?;
+
+        // connection
+        let mut connection = Imap::new_configured(self, channel::bounded(1).1).await?;
+        let mut session = connection.prepare(self).await?;
+
+        // fetch imap folders
+        for folder_meaning in [FolderMeaning::Inbox, FolderMeaning::Mvbox] {
+            let (_, watch_folder) = convert_folder_meaning(self, folder_meaning).await?;
+            connection
+                .fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
+                .await?;
+        }
+
+        // update quota (to send warning if full) - but only check it once in a while
+        let quota_needs_update = {
+            let quota = self.quota.read().await;
+            quota
+                .as_ref()
+                .filter(|quota| {
+                    time_elapsed(&quota.modified)
+                        > Duration::from_secs(DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT)
+                })
+                .is_none()
+        };
+
+        if quota_needs_update {
+            if let Err(err) = self.update_recent_quota(&mut session).await {
+                warn!(self, "Failed to update quota: {err:#}.");
+            }
+        }
+
+        info!(
+            self,
+            "background_fetch done for {address} took {:?}",
+            time_elapsed(&time_start),
+        );
+
+        Ok(())
+    }
+
     pub(crate) async fn schedule_resync(&self) -> Result<()> {
         self.resync_request.store(true, Ordering::Relaxed);
         self.scheduler.interrupt_inbox().await;
@@ -551,7 +640,7 @@ impl Context {
     pub(crate) async fn free_ongoing(&self) {
         let mut s = self.running_state.write().await;
         if let RunningState::ShallStop { request } = *s {
-            info!(self, "Ongoing stopped in {:?}", request.elapsed());
+            info!(self, "Ongoing stopped in {:?}", time_elapsed(&request));
         }
         *s = RunningState::Stopped;
     }
@@ -566,7 +655,7 @@ impl Context {
                 }
                 info!(self, "Signaling the ongoing process to stop ASAP.",);
                 *s = RunningState::ShallStop {
-                    request: Instant::now(),
+                    request: tools::Time::now(),
                 };
             }
             RunningState::ShallStop { .. } | RunningState::Stopped => {
@@ -632,7 +721,7 @@ impl Context {
         let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
         let folders_configured = self
             .sql
-            .get_raw_config_int("folders_configured")
+            .get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
             .await?
             .unwrap_or_default();
 
@@ -672,7 +761,7 @@ impl Context {
         );
         res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
-        res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
+        res.insert("displayname", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
@@ -688,6 +777,16 @@ impl Context {
             res.insert("imap_server_id", format!("{server_id:?}"));
         }
 
+        if let Some(metadata) = &*self.metadata.read().await {
+            if let Some(comment) = &metadata.comment {
+                res.insert("imap_server_comment", format!("{comment:?}"));
+            }
+
+            if let Some(admin) = &metadata.admin {
+                res.insert("imap_server_admin", format!("{admin:?}"));
+            }
+        }
+
         res.insert("secondary_addrs", secondary_addrs);
         res.insert(
             "fetch_existing_msgs",
@@ -714,7 +813,10 @@ impl Context {
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_move", mvbox_move.to_string());
         res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
-        res.insert("folders_configured", folders_configured.to_string());
+        res.insert(
+            constants::DC_FOLDERS_CONFIGURED_KEY,
+            folders_configured.to_string(),
+        );
         res.insert("configured_inbox_folder", configured_inbox_folder);
         res.insert("configured_sentbox_folder", configured_sentbox_folder);
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
@@ -765,6 +867,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "last_cant_decrypt_outgoing_msgs",
+            self.get_config_int(Config::LastCantDecryptOutgoingMsgs)
+                .await?
+                .to_string(),
+        );
         res.insert(
             "scan_all_folders_debounce_secs",
             self.get_config_int(Config::ScanAllFoldersDebounceSecs)
@@ -808,12 +916,177 @@ impl Context {
                 .to_string(),
         );
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
+        let elapsed = time_elapsed(&self.creation_time);
+        res.insert("uptime", duration_to_str(elapsed));
 
         Ok(res)
     }
 
+    async fn get_self_report(&self) -> Result<String> {
+        #[derive(Default)]
+        struct ChatNumbers {
+            protected: u32,
+            protection_broken: u32,
+            opportunistic_dc: u32,
+            opportunistic_mua: u32,
+            unencrypted_dc: u32,
+            unencrypted_mua: u32,
+        }
+
+        let mut res = String::new();
+        res += &format!("core_version {}\n", get_version_str());
+
+        let num_msgs: u32 = self
+            .sql
+            .query_get_value(
+                "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?",
+                (DC_CHAT_ID_TRASH,),
+            )
+            .await?
+            .unwrap_or_default();
+        res += &format!("num_msgs {}\n", num_msgs);
+
+        let num_chats: u32 = self
+            .sql
+            .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ())
+            .await?
+            .unwrap_or_default();
+        res += &format!("num_chats {}\n", num_chats);
+
+        let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len();
+        res += &format!("db_size_bytes {}\n", db_size);
+
+        let secret_key = &load_self_secret_key(self).await?.primary_key;
+        let key_created = secret_key.created_at().timestamp();
+        res += &format!("key_created {}\n", key_created);
+
+        // how many of the chats active in the last months are:
+        // - protected
+        // - protection-broken
+        // - opportunistic-encrypted and the contact uses Delta Chat
+        // - opportunistic-encrypted and the contact uses a classical MUA
+        // - unencrypted and the contact uses Delta Chat
+        // - unencrypted and the contact uses a classical MUA
+        let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3);
+        let chats = self
+            .sql
+            .query_map(
+                "SELECT c.protected, m.param, m.msgrmsg
+                    FROM chats c
+                    JOIN msgs m
+                        ON c.id=m.chat_id
+                        AND m.id=(
+                                SELECT id
+                                FROM msgs
+                                WHERE chat_id=c.id
+                                AND hidden=0
+                                AND download_state=?
+                                AND to_id!=?
+                                ORDER BY timestamp DESC, id DESC LIMIT 1)
+                    WHERE c.id>9
+                    AND (c.blocked=0 OR c.blocked=2)
+                    AND IFNULL(m.timestamp,c.created_timestamp) > ?
+                    GROUP BY c.id",
+                (DownloadState::Done, ContactId::INFO, three_months_ago),
+                |row| {
+                    let protected: ProtectionStatus = row.get(0)?;
+                    let message_param: Params =
+                        row.get::<_, String>(1)?.parse().unwrap_or_default();
+                    let is_dc_message: bool = row.get(2)?;
+                    Ok((protected, message_param, is_dc_message))
+                },
+                |rows| {
+                    let mut chats = ChatNumbers::default();
+                    for row in rows {
+                        let (protected, message_param, is_dc_message) = row?;
+                        let encrypted = message_param
+                            .get_bool(Param::GuaranteeE2ee)
+                            .unwrap_or(false);
+
+                        if protected == ProtectionStatus::Protected {
+                            chats.protected += 1;
+                        } else if protected == ProtectionStatus::ProtectionBroken {
+                            chats.protection_broken += 1;
+                        } else if encrypted {
+                            if is_dc_message {
+                                chats.opportunistic_dc += 1;
+                            } else {
+                                chats.opportunistic_mua += 1;
+                            }
+                        } else if is_dc_message {
+                            chats.unencrypted_dc += 1;
+                        } else {
+                            chats.unencrypted_mua += 1;
+                        }
+                    }
+                    Ok(chats)
+                },
+            )
+            .await?;
+        res += &format!("chats_protected {}\n", chats.protected);
+        res += &format!("chats_protection_broken {}\n", chats.protection_broken);
+        res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc);
+        res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua);
+        res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc);
+        res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua);
+
+        let self_reporting_id = match self.get_config(Config::SelfReportingId).await? {
+            Some(id) => id,
+            None => {
+                let id = create_id();
+                self.set_config(Config::SelfReportingId, Some(&id)).await?;
+                id
+            }
+        };
+        res += &format!("self_reporting_id {}", self_reporting_id);
+
+        Ok(res)
+    }
+
+    /// Drafts a message with statistics about the usage of Delta Chat.
+    /// The user can inspect the message if they want, and then hit "Send".
+    ///
+    /// On the other end, a bot will receive the message and make it available
+    /// to Delta Chat's developers.
+    pub async fn draft_self_report(&self) -> Result<ChatId> {
+        const SELF_REPORTING_BOT: &str = "self_reporting@testrun.org";
+
+        let contact_id = Contact::create(self, "Statistics bot", SELF_REPORTING_BOT).await?;
+        let chat_id = ChatId::create_for_contact(self, contact_id).await?;
+
+        // We're including the bot's public key in Delta Chat
+        // so that the first message to the bot can directly be encrypted:
+        let public_key = SignedPublicKey::from_base64(
+            "xjMEZbfBlBYJKwYBBAHaRw8BAQdABpLWS2PUIGGo4pslVt4R8sylP5wZihmhf1DTDr3oCM\
+	        PNHDxzZWxmX3JlcG9ydGluZ0B0ZXN0cnVuLm9yZz7CiwQQFggAMwIZAQUCZbfBlAIbAwQLCQgHBhUI\
+	        CQoLAgMWAgEWIQTS2i16sHeYTckGn284K3M5Z4oohAAKCRA4K3M5Z4oohD8dAQCQV7CoH6UP4PD+Nq\
+	        I4kW5tbbqdh2AnDROg60qotmLExAEAxDfd3QHAK9f8b9qQUbLmHIztCLxhEuVbWPBEYeVW0gvOOARl\
+	        t8GUEgorBgEEAZdVAQUBAQdAMBUhYoAAcI625vGZqnM5maPX4sGJ7qvJxPAFILPy6AcDAQgHwngEGB\
+	        YIACAFAmW3wZQCGwwWIQTS2i16sHeYTckGn284K3M5Z4oohAAKCRA4K3M5Z4oohPwCAQCvzk1ObIkj\
+	        2GqsuIfaULlgdnfdZY8LNary425CEfHZDQD5AblXVrlMO1frdlc/Vo9z3pEeCrfYdD7ITD3/OeVoiQ\
+	        4=",
+        )?;
+        let mut peerstate = Peerstate::from_public_key(
+            SELF_REPORTING_BOT,
+            0,
+            EncryptPreference::Mutual,
+            &public_key,
+        );
+        let fingerprint = public_key.fingerprint();
+        peerstate.set_verified(public_key, fingerprint, "".to_string())?;
+        peerstate.save_to_db(&self.sql).await?;
+        chat_id
+            .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id))
+            .await?;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = self.get_self_report().await?;
+
+        chat_id.set_draft(self, Some(&mut msg)).await?;
+
+        Ok(chat_id)
+    }
+
     /// Get a list of fresh, unmuted messages in unblocked chats.
     ///
     /// The list starts with the most recent message
@@ -1084,9 +1357,10 @@ mod tests {
     use crate::constants::Chattype;
     use crate::contact::ContactId;
     use crate::message::{Message, Viewtype};
+    use crate::mimeparser::SystemMessage;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::TestContext;
-    use crate::tools::create_outgoing_rfc724_mid;
+    use crate::test_utils::{get_chat_msg, TestContext};
+    use crate::tools::{create_outgoing_rfc724_mid, SystemTime};
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_wrong_db() -> Result<()> {
@@ -1121,7 +1395,7 @@ mod tests {
              \n\
              hello\n",
             contact.get_addr(),
-            create_outgoing_rfc724_mid(None, contact.get_addr())
+            create_outgoing_rfc724_mid(contact.get_addr())
         );
         println!("{msg}");
         receive_imf(t, msg.as_bytes(), false).await.unwrap();
@@ -1271,7 +1545,14 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new(), StockStrings::new());
+        let res = Context::with_blobdir(
+            dbfile,
+            blobdir,
+            1,
+            Events::new(),
+            StockStrings::new(),
+            Default::default(),
+        );
         assert!(res.is_err());
     }
 
@@ -1280,7 +1561,14 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(dbfile, blobdir, 1, Events::new(), StockStrings::new());
+        let res = Context::with_blobdir(
+            dbfile,
+            blobdir,
+            1,
+            Events::new(),
+            StockStrings::new(),
+            Default::default(),
+        );
         assert!(res.is_err());
     }
 
@@ -1324,6 +1612,7 @@ mod tests {
             "mail_security",
             "notify_about_wrong_pw",
             "save_mime_headers",
+            "self_reporting_id",
             "selfstatus",
             "send_server",
             "send_user",
@@ -1338,7 +1627,6 @@ mod tests {
             "socks5_user",
             "socks5_password",
             "key_id",
-            "iroh_secret_key",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -1503,16 +1791,18 @@ mod tests {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
-        let id = 1;
-        let context = Context::new_closed(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile.clone())
+            .with_id(1)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);
         assert_eq!(context.is_open().await, true);
         drop(context);
 
-        let id = 2;
-        let context = Context::new(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile)
+            .with_id(2)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.is_open().await, false);
@@ -1528,8 +1818,9 @@ mod tests {
         let dir = tempdir()?;
         let dbfile = dir.path().join("db.sqlite");
 
-        let id = 1;
-        let context = Context::new_closed(&dbfile, id, Events::new(), StockStrings::new())
+        let context = ContextBuilder::new(dbfile)
+            .with_id(1)
+            .build()
             .await
             .context("failed to create context")?;
         assert_eq!(context.open("foo".to_string()).await?, true);
@@ -1627,12 +1918,21 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_keypair_saving() -> Result<()> {
+    async fn test_draft_self_report() -> Result<()> {
         let alice = TestContext::new_alice().await;
 
-        let key = alice.get_or_generate_iroh_keypair().await?;
-        let loaded_key = alice.get_or_generate_iroh_keypair().await?;
-        assert_eq!(key.to_bytes(), loaded_key.to_bytes());
+        let chat_id = alice.draft_self_report().await?;
+        let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
+        assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
+
+        let chat = Chat::load_from_db(&alice, chat_id).await?;
+        assert!(chat.is_protected());
+
+        let mut draft = chat_id.get_draft(&alice).await?.unwrap();
+        assert!(draft.text.starts_with("core_version"));
+
+        // Test that sending into the protected chat works:
+        let _sent = alice.send_msg(chat_id, &mut draft).await;
 
         Ok(())
     }

src/debug_logging.rs

@@ -63,7 +63,6 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                     summary: None,
                     document: None,
                     uid: None,
-                    gossip_topic: None,
                 },
             )
             .await
@@ -73,10 +72,12 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
             }
             Ok(serial) => {
                 if let Some(serial) = serial {
-                    context.emit_event(EventType::WebxdcStatusUpdate {
-                        msg_id,
-                        status_update_serial: serial,
-                    });
+                    if !matches!(event, EventType::WebxdcStatusUpdate { .. }) {
+                        context.emit_event(EventType::WebxdcStatusUpdate {
+                            msg_id,
+                            status_update_serial: serial,
+                        });
+                    }
                 } else {
                     // This should not happen as the update has no `uid`.
                     error!(context, "Debug logging update is not created.");

src/decrypt.rs

@@ -12,7 +12,7 @@ use crate::authres::{self, DkimResults};
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
+use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::peerstate::Peerstate;
 use crate::pgp;
 
@@ -264,16 +264,22 @@ pub(crate) fn validate_detached_signature<'a, 'b>(
     }
 }
 
-pub(crate) fn keyring_from_peerstate(peerstate: Option<&Peerstate>) -> Vec<SignedPublicKey> {
+/// Returns public keyring for `peerstate`.
+pub(crate) async fn keyring_from_peerstate(
+    context: &Context,
+    peerstate: Option<&Peerstate>,
+) -> Result<Vec<SignedPublicKey>> {
     let mut public_keyring_for_validate = Vec::new();
     if let Some(peerstate) = peerstate {
         if let Some(key) = &peerstate.public_key {
             public_keyring_for_validate.push(key.clone());
         } else if let Some(key) = &peerstate.gossip_key {
             public_keyring_for_validate.push(key.clone());
+        } else if context.is_self_addr(&peerstate.addr).await? {
+            public_keyring_for_validate = key::load_self_public_keyring(context).await?;
         }
     }
-    public_keyring_for_validate
+    Ok(public_keyring_for_validate)
 }
 
 /// Applies Autocrypt header to Autocrypt peer state and saves it into the database.
@@ -292,6 +298,7 @@ pub(crate) async fn get_autocrypt_peerstate(
     message_time: i64,
     allow_change: bool,
 ) -> Result<Option<Peerstate>> {
+    let allow_change = allow_change && !context.is_self_addr(from).await?;
     let mut peerstate;
 
     // Apply Autocrypt header

src/dehtml.rs

@@ -181,7 +181,12 @@ fn dehtml_text_cb(event: &BytesText, dehtml: &mut Dehtml) {
     if dehtml.get_add_text() == AddText::YesPreserveLineEnds
         || dehtml.get_add_text() == AddText::YesRemoveLineEnds
     {
-        let last_added = escaper::decode_html_buf_sloppy(event as &[_]).unwrap_or_default();
+        let event = event as &[_];
+        let event_str = std::str::from_utf8(event).unwrap_or_default();
+        let mut last_added = escaper::decode_html_buf_sloppy(event).unwrap_or_default();
+        if event_str.starts_with(&last_added) {
+            last_added = event_str.to_string();
+        }
 
         if dehtml.get_add_text() == AddText::YesRemoveLineEnds {
             // Replace all line ends with spaces.
@@ -527,6 +532,6 @@ mod tests {
     fn test_spaces() {
         let input = include_str!("../test-data/spaces.html");
         let txt = dehtml(input).unwrap();
-        assert_eq!(txt.text, "Welcome back to Strolling!\n\nHey there,\n\nWelcome back! Use this link to securely sign in to your Strolling account:\n\nSign in to Strolling\n\nFor your security, the link will expire in 24 hours time.\n\nSee you soon!\n\nYou can also copy\n\nhttps://strolling.rosano.ca/members/?token=XXX\n\nIf you did not make this request, you can safely ignore this email.\n\nThis message was sent from [strolling.rosano.ca](https://strolling.rosano.ca/) to [alice@example.org](mailto:alice@example.org)");
+        assert_eq!(txt.text, "Welcome back to Strolling!\n\nHey there,\n\nWelcome back! Use this link to securely sign in to your Strolling account:\n\nSign in to Strolling\n\nFor your security, the link will expire in 24 hours time.\n\nSee you soon!\n\nYou can also copy & paste this URL into your browser:\n\nhttps://strolling.rosano.ca/members/?token=XXX&action=signin&r=https%3A%2F%2Fstrolling.rosano.ca%2F\n\nIf you did not make this request, you can safely ignore this email.\n\nThis message was sent from [strolling.rosano.ca](https://strolling.rosano.ca/) to [alice@example.org](mailto:alice@example.org)");
     }
 }

src/download.rs

@@ -3,13 +3,13 @@
 use std::cmp::max;
 use std::collections::BTreeMap;
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 
 use crate::config::Config;
 use crate::context::Context;
-use crate::imap::{Imap, ImapActionResult};
+use crate::imap::session::Session;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
 use crate::tools::time;
@@ -129,40 +129,44 @@ impl Message {
 /// Actually download a message partially downloaded before.
 ///
 /// Most messages are downloaded automatically on fetch instead.
-pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Imap) -> Result<()> {
-    imap.prepare(context).await?;
-
+pub(crate) async fn download_msg(
+    context: &Context,
+    msg_id: MsgId,
+    session: &mut Session,
+) -> Result<()> {
     let msg = Message::load_from_db(context, msg_id).await?;
     let row = context
         .sql
         .query_row_optional(
-            "SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
+            "SELECT uid, folder, uidvalidity FROM imap WHERE rfc724_mid=? AND target!=''",
             (&msg.rfc724_mid,),
             |row| {
                 let server_uid: u32 = row.get(0)?;
                 let server_folder: String = row.get(1)?;
-                Ok((server_uid, server_folder))
+                let uidvalidity: u32 = row.get(2)?;
+                Ok((server_uid, server_folder, uidvalidity))
             },
         )
         .await?;
 
-    let Some((server_uid, server_folder)) = row else {
+    let Some((server_uid, server_folder, uidvalidity)) = row else {
         // No IMAP record found, we don't know the UID and folder.
         return Err(anyhow!("Call download_full() again to try over."));
     };
 
-    match imap
-        .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
-        .await
-    {
-        ImapActionResult::RetryLater | ImapActionResult::Failed => {
-            Err(anyhow!("Call download_full() again to try over."))
-        }
-        ImapActionResult::Success => Ok(()),
-    }
+    session
+        .fetch_single_msg(
+            context,
+            &server_folder,
+            uidvalidity,
+            server_uid,
+            msg.rfc724_mid.clone(),
+        )
+        .await?;
+    Ok(())
 }
 
-impl Imap {
+impl Session {
     /// Download a single message and pipe it to receive_imf().
     ///
     /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
@@ -171,33 +175,36 @@ impl Imap {
         &mut self,
         context: &Context,
         folder: &str,
+        uidvalidity: u32,
         uid: u32,
         rfc724_mid: String,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
+    ) -> Result<()> {
+        if uid == 0 {
+            bail!("Attempt to fetch UID 0");
         }
 
+        self.select_folder(context, Some(folder)).await?;
+
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
 
         let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
         uid_message_ids.insert(uid, rfc724_mid);
-        let (last_uid, _received) = match self
-            .fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, false)
-            .await
-        {
-            Ok(res) => res,
-            Err(_) => return ImapActionResult::Failed,
-        };
+        let (last_uid, _received) = self
+            .fetch_many_msgs(
+                context,
+                folder,
+                uidvalidity,
+                vec![uid],
+                &uid_message_ids,
+                false,
+                false,
+            )
+            .await?;
         if last_uid.is_none() {
-            ImapActionResult::Failed
-        } else {
-            ImapActionResult::Success
+            bail!("Failed to fetch UID {uid}");
         }
+        Ok(())
     }
 }
 
@@ -247,7 +254,7 @@ mod tests {
     use crate::chat::{get_chat_msgs, send_msg};
     use crate::ephemeral::Timer;
     use crate::message::Viewtype;
-    use crate::receive_imf::receive_imf_inner;
+    use crate::receive_imf::receive_imf_from_inbox;
     use crate::test_utils::TestContext;
 
     #[test]
@@ -328,7 +335,7 @@ mod tests {
              Date: Sun, 22 Mar 2020 22:37:57 +0000\
              Content-Type: text/plain";
 
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "Mr.12345678901@example.com",
             header.as_bytes(),
@@ -344,7 +351,7 @@ mod tests {
             .get_text()
             .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
 
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "Mr.12345678901@example.com",
             format!("{header}\n\n100k text...").as_bytes(),
@@ -373,7 +380,7 @@ mod tests {
             .await?;
 
         // download message from bob partially, this must not change the ephemeral timer
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &t,
             "first@example.org",
             b"From: Bob <bob@example.org>\n\
@@ -416,7 +423,7 @@ mod tests {
         let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid;
 
         // not downloading the status update results in an placeholder
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             &sent2_rfc724_mid,
             sent2.payload().as_bytes(),
@@ -432,7 +439,7 @@ mod tests {
 
         // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
         // (usually status updates are too small for not being downloaded directly)
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             &sent2_rfc724_mid,
             sent2.payload().as_bytes(),
@@ -483,7 +490,7 @@ mod tests {
             ";
 
         // not downloading the mdn results in an placeholder
-        receive_imf_inner(
+        receive_imf_from_inbox(
             &bob,
             "bar@example.org",
             raw,
@@ -499,7 +506,7 @@ mod tests {
 
         // downloading the mdn afterwards expands to nothing and deletes the placeholder directly
         // (usually mdn are too small for not being downloaded directly)
-        receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?;
+        receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None, false).await?;
         assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0);
         assert!(Message::load_from_db(&bob, msg.id)
             .await?

src/e2ee.rs

@@ -95,6 +95,7 @@ impl EncryptHelper {
         verified: bool,
         mail_to_encrypt: lettre_email::PartBuilder,
         peerstates: Vec<(Option<Peerstate>, String)>,
+        compress: bool,
     ) -> Result<String> {
         let mut keyring: Vec<SignedPublicKey> = Vec::new();
 
@@ -135,7 +136,7 @@ impl EncryptHelper {
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key), compress).await?;
 
         Ok(ctext)
     }

src/ephemeral.rs

@@ -67,7 +67,7 @@ use std::collections::BTreeSet;
 use std::convert::{TryFrom, TryInto};
 use std::num::ParseIntError;
 use std::str::FromStr;
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use std::time::{Duration, UNIX_EPOCH};
 
 use anyhow::{ensure, Result};
 use async_channel::Receiver;
@@ -85,7 +85,7 @@ use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
 use crate::sql::{self, params_iter};
 use crate::stock_str;
-use crate::tools::{duration_to_str, time};
+use crate::tools::{duration_to_str, time, SystemTime};
 
 /// Ephemeral timer value.
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
@@ -590,7 +590,11 @@ pub(crate) async fn delete_expired_imap_messages(context: &Context) -> Result<()
         match context.get_config_delete_server_after().await? {
             None => (0, 0),
             Some(delete_server_after) => (
-                now - delete_server_after,
+                match delete_server_after {
+                    // Guarantee immediate deletion.
+                    0 => i64::MAX,
+                    _ => now - delete_server_after,
+                },
                 now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
             ),
         };
@@ -1127,6 +1131,7 @@ mod tests {
             (1030, now - 19 * HOUR, 0),
             (2000, now - 18 * HOUR, now - HOUR),
             (2020, now - 17 * HOUR, now + HOUR),
+            (3000, now + HOUR, 0),
         ] {
             let message_id = id.to_string();
             t.sql
@@ -1212,6 +1217,10 @@ mod tests {
             0
         );
 
+        t.set_config(Config::DeleteServerAfter, Some("1")).await?;
+        delete_expired_imap_messages(&t).await?;
+        test_marked_for_deletion(&t, 3000).await?;
+
         Ok(())
     }
 

src/events/payload.rs

@@ -287,4 +287,11 @@ pub enum EventType {
         /// ID of the deleted message.
         msg_id: MsgId,
     },
+
+    /// Tells that the Background fetch was completed (or timed out).
+    /// This event acts as a marker, when you reach this event you can be sure
+    /// that all events emitted during the background fetch were processed.
+    ///
+    /// This event is only emitted by the account manager
+    AccountsBackgroundFetchDone,
 }

src/headerdef.rs

@@ -2,7 +2,7 @@
 
 use mailparse::{MailHeader, MailHeaderMap};
 
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, IntoStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, IntoStaticStr)]
 #[strum(serialize_all = "kebab_case")]
 #[allow(missing_docs)]
 pub enum HeaderDef {
@@ -74,6 +74,11 @@ pub enum HeaderDef {
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
+
+    /// Deprecated header containing Group-ID in `vg-request-with-auth` message.
+    ///
+    /// It is not used by Alice as Alice knows the group corresponding to the AUTH token.
+    /// Bob still sends it for backwards compatibility.
     SecureJoinGroup,
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
@@ -88,9 +93,6 @@ pub enum HeaderDef {
     /// See <https://datatracker.ietf.org/doc/html/rfc8601>
     AuthenticationResults,
 
-    /// Public key to join gossip network.
-    IrohPublicGossip,
-
     #[cfg(test)]
     TestHeader,
 }

src/imap/capabilities.rs

@@ -21,6 +21,17 @@ pub(crate) struct Capabilities {
     /// <https://tools.ietf.org/html/rfc7162>
     pub can_condstore: bool,
 
+    /// True if the server has METADATA capability as defined in
+    /// <https://tools.ietf.org/html/rfc5464>
+    pub can_metadata: bool,
+
+    /// True if the server supports XDELTAPUSH capability.
+    /// This capability means setting /private/devicetoken IMAP METADATA
+    /// on the INBOX results in new mail notifications
+    /// via notifications.delta.chat service.
+    /// This is supported by <https://github.com/deltachat/chatmail>
+    pub can_push: bool,
+
     /// Server ID if the server supports ID capability.
     pub server_id: Option<HashMap<String, String>>,
 }

src/imap/client.rs

@@ -59,6 +59,8 @@ async fn determine_capabilities(
         can_move: caps.has_str("MOVE"),
         can_check_quota: caps.has_str("QUOTA"),
         can_condstore: caps.has_str("CONDSTORE"),
+        can_metadata: caps.has_str("METADATA"),
+        can_push: caps.has_str("XDELTAPUSH"),
         server_id,
     };
     Ok(capabilities)

src/imap/idle.rs

@@ -1,16 +1,16 @@
-use std::time::{Duration, SystemTime};
+use std::time::Duration;
 
 use anyhow::{bail, Context as _, Result};
 use async_channel::Receiver;
 use async_imap::extensions::idle::IdleResponse;
 use futures_lite::FutureExt;
+use tokio::time::timeout;
 
 use super::session::Session;
 use super::Imap;
-use crate::config::Config;
 use crate::context::Context;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
-use crate::log::LogExt;
+use crate::tools::{self, time_elapsed};
 
 /// Timeout after which IDLE is finished
 /// if there are no responses from the server.
@@ -97,97 +97,38 @@ impl Session {
 }
 
 impl Imap {
+    /// Idle using polling.
     pub(crate) async fn fake_idle(
         &mut self,
         context: &Context,
-        watch_folder: Option<String>,
+        session: &mut Session,
+        watch_folder: String,
         folder_meaning: FolderMeaning,
-    ) {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
+    ) -> Result<()> {
+        let fake_idle_start_time = tools::Time::now();
 
-        let fake_idle_start_time = SystemTime::now();
-
-        // Do not poll, just wait for an interrupt when no folder is passed in.
-        let watch_folder = if let Some(watch_folder) = watch_folder {
-            watch_folder
-        } else {
-            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
-            self.idle_interrupt_receiver.recv().await.ok();
-            return;
-        };
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        const TIMEOUT_INIT_MS: u64 = 60_000;
-        let mut timeout_ms: u64 = TIMEOUT_INIT_MS;
-        enum Event {
-            Tick,
-            Interrupt,
-        }
-        // loop until we are interrupted or if we fetched something
+        // Loop until we are interrupted or until we fetch something.
         loop {
-            use futures::future::FutureExt;
-            use rand::Rng;
-
-            let mut interval = tokio::time::interval(Duration::from_millis(timeout_ms));
-            timeout_ms = timeout_ms
-                .saturating_add(rand::thread_rng().gen_range((timeout_ms / 2)..=timeout_ms));
-            interval.tick().await; // The first tick completes immediately.
-            match interval
-                .tick()
-                .map(|_| Event::Tick)
-                .race(
-                    self.idle_interrupt_receiver
-                        .recv()
-                        .map(|_| Event::Interrupt),
-                )
-                .await
-            {
-                Event::Tick => {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.prepare(context).await {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if let Some(session) = &self.session {
-                        if session.can_idle()
-                            && !context
-                                .get_config_bool(Config::DisableIdle)
-                                .await
-                                .context("Failed to get disable_idle config")
-                                .log_err(context)
-                                .unwrap_or_default()
-                        {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
+            match timeout(Duration::from_secs(60), self.idle_interrupt_receiver.recv()).await {
+                Err(_) => {
+                    // Let's see if fetching messages results
                     // in anything.  If so, we behave as if IDLE had data but
                     // will have already fetched the messages so perform_*_fetch
                     // will not find any new.
-                    match self
-                        .fetch_new_messages(context, &watch_folder, folder_meaning, false)
-                        .await
-                    {
-                        Ok(res) => {
-                            info!(context, "fetch_new_messages returned {:?}", res);
-                            timeout_ms = TIMEOUT_INIT_MS;
-                            if res {
-                                break;
-                            }
-                        }
-                        Err(err) => {
-                            error!(context, "could not fetch from folder: {:#}", err);
-                            self.trigger_reconnect(context);
-                        }
+                    let res = self
+                        .fetch_new_messages(context, session, &watch_folder, folder_meaning, false)
+                        .await?;
+
+                    info!(context, "fetch_new_messages returned {:?}", res);
+
+                    if res {
+                        break;
                     }
                 }
-                Event::Interrupt => {
-                    info!(context, "Fake IDLE interrupted");
+                Ok(_) => {
+                    info!(context, "Fake IDLE interrupted.");
                     break;
                 }
             }
@@ -196,11 +137,8 @@ impl Imap {
         info!(
             context,
             "IMAP-fake-IDLE done after {:.4}s",
-            SystemTime::now()
-                .duration_since(fake_idle_start_time)
-                .unwrap_or_default()
-                .as_millis() as f64
-                / 1000.,
+            time_elapsed(&fake_idle_start_time).as_millis() as f64 / 1000.,
         );
+        Ok(())
     }
 }

src/imap/scan_folders.rs

@@ -1,21 +1,25 @@
-use std::{collections::BTreeMap, time::Instant};
+use std::collections::BTreeMap;
 
 use anyhow::{Context as _, Result};
-use futures::TryStreamExt;
 
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
-use crate::imap::Imap;
+use crate::imap::{session::Session, Imap};
 use crate::log::LogExt;
+use crate::tools::{self, time_elapsed};
 use crate::{context::Context, imap::FolderMeaning};
 
 impl Imap {
     /// Returns true if folders were scanned, false if scanning was postponed.
-    pub(crate) async fn scan_folders(&mut self, context: &Context) -> Result<bool> {
+    pub(crate) async fn scan_folders(
+        &mut self,
+        context: &Context,
+        session: &mut Session,
+    ) -> Result<bool> {
         // First of all, debounce to once per minute:
         let mut last_scan = context.last_full_folder_scan.lock().await;
         if let Some(last_scan) = *last_scan {
-            let elapsed_secs = last_scan.elapsed().as_secs();
+            let elapsed_secs = time_elapsed(&last_scan).as_secs();
             let debounce_secs = context
                 .get_config_u64(Config::ScanAllFoldersDebounceSecs)
                 .await?;
@@ -26,8 +30,7 @@ impl Imap {
         }
         info!(context, "Starting full folder scan");
 
-        self.prepare(context).await?;
-        let folders = self.list_folders().await?;
+        let folders = session.list_folders().await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
@@ -63,18 +66,16 @@ impl Imap {
                 && folder_meaning != FolderMeaning::Drafts
                 && folder_meaning != FolderMeaning::Trash
             {
-                let session = self.session.as_mut().context("no session")?;
                 // Drain leftover unsolicited EXISTS messages
                 session.server_sent_unsolicited_exists(context)?;
 
                 loop {
-                    self.fetch_move_delete(context, folder.name(), folder_meaning)
+                    self.fetch_move_delete(context, session, folder.name(), folder_meaning)
                         .await
                         .context("Can't fetch new msgs in scanned folder")
                         .log_err(context)
                         .ok();
 
-                    let session = self.session.as_mut().context("no session")?;
                     // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
                     if !session.server_sent_unsolicited_exists(context)? {
                         break;
@@ -89,25 +90,13 @@ impl Imap {
             Config::ConfiguredTrashFolder,
         ] {
             context
-                .set_config(conf, folder_configs.get(&conf).map(|s| s.as_str()))
+                .set_config_internal(conf, folder_configs.get(&conf).map(|s| s.as_str()))
                 .await?;
         }
 
-        last_scan.replace(Instant::now());
+        last_scan.replace(tools::Time::now());
         Ok(true)
     }
-
-    /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
-        let session = self.session.as_mut();
-        let session = session.context("No IMAP connection")?;
-        let list = session
-            .list(Some(""), Some("*"))
-            .await?
-            .try_collect()
-            .await?;
-        Ok(list)
-    }
 }
 
 pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {

src/imap/select_folder.rs

@@ -3,6 +3,7 @@
 use anyhow::Context as _;
 
 use super::session::Session as ImapSession;
+use super::{get_uid_next, get_uidvalidity, set_modseq, set_uid_next, set_uidvalidity};
 use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
@@ -51,7 +52,7 @@ impl ImapSession {
     /// Selects a folder, possibly updating uid_validity and, if needed,
     /// expunging the folder to remove delete-marked messages.
     /// Returns whether a new folder was selected.
-    pub(super) async fn select_folder(
+    pub(crate) async fn select_folder(
         &mut self,
         context: &Context,
         folder: Option<&str>,
@@ -112,20 +113,150 @@ impl ImapSession {
             Err(err) => match err {
                 Error::NoFolder(..) => {
                     info!(context, "Failed to select folder {} because it does not exist, trying to create it.", folder);
-                    self.create(folder).await.with_context(|| {
-                        format!("Couldn't select folder ('{err}'), then create() failed")
-                    })?;
-
-                    Ok(self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {folder}"))?)
+                    let create_res = self.create(folder).await;
+                    if let Err(ref err) = create_res {
+                        info!(context, "Couldn't select folder, then create() failed: {err:#}.");
+                        // Need to recheck, could have been created in parallel.
+                    }
+                    let select_res = self.select_folder(context, Some(folder)).await.with_context(|| format!("failed to select newely created folder {folder}"));
+                    if select_res.is_err() {
+                        create_res?;
+                    }
+                    select_res
                 }
                 _ => Err(err).with_context(|| format!("failed to select folder {folder} with error other than NO, not trying to create it")),
             },
         }
     }
+
+    /// Selects a folder and takes care of UIDVALIDITY changes.
+    ///
+    /// When selecting a folder for the first time, sets the uid_next to the current
+    /// mailbox.uid_next so that no old emails are fetched.
+    ///
+    /// Returns Result<new_emails> (i.e. whether new emails arrived),
+    /// if in doubt, returns new_emails=true so emails are fetched.
+    pub(crate) async fn select_with_uidvalidity(
+        &mut self,
+        context: &Context,
+        folder: &str,
+    ) -> Result<bool> {
+        let newly_selected = self
+            .select_or_create_folder(context, folder)
+            .await
+            .with_context(|| format!("failed to select or create folder {folder}"))?;
+        let mailbox = self
+            .selected_mailbox
+            .as_mut()
+            .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
+
+        let old_uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
+        let old_uid_next = get_uid_next(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+
+        let new_uid_validity = mailbox
+            .uid_validity
+            .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
+        let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
+            Some(uid_next)
+        } else {
+            warn!(
+                context,
+                "SELECT response for IMAP folder {folder:?} has no UIDNEXT, fall back to STATUS command."
+            );
+
+            // RFC 3501 says STATUS command SHOULD NOT be used
+            // on the currently selected mailbox because the same
+            // information can be obtained by other means,
+            // such as reading SELECT response.
+            //
+            // However, it also says that UIDNEXT is REQUIRED
+            // in the SELECT response and if we are here,
+            // it is actually not returned.
+            //
+            // In particular, Winmail Pro Mail Server 5.1.0616
+            // never returns UIDNEXT in SELECT response,
+            // but responds to "STATUS INBOX (UIDNEXT)" command.
+            let status = self
+                .inner
+                .status(folder, "(UIDNEXT)")
+                .await
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
+
+            if status.uid_next.is_none() {
+                // This happens with mail.163.com as of 2023-11-26.
+                // It does not return UIDNEXT on SELECT and returns invalid
+                // `* STATUS "INBOX" ()` response on explicit request for UIDNEXT.
+                warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT.");
+            }
+            status.uid_next
+        };
+        mailbox.uid_next = new_uid_next;
+
+        if new_uid_validity == old_uid_validity {
+            let new_emails = if newly_selected == NewlySelected::No {
+                // The folder was not newly selected i.e. no SELECT command was run. This means that mailbox.uid_next
+                // was not updated and may contain an incorrect value. So, just return true so that
+                // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
+                // new messages is only one command, just as a SELECT command)
+                true
+            } else if let Some(new_uid_next) = new_uid_next {
+                if new_uid_next < old_uid_next {
+                    warn!(
+                        context,
+                        "The server illegally decreased the uid_next of folder {folder:?} from {old_uid_next} to {new_uid_next} without changing validity ({new_uid_validity}), resyncing UIDs...",
+                    );
+                    set_uid_next(context, folder, new_uid_next).await?;
+                    context.schedule_resync().await?;
+                }
+                new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
+            } else {
+                // We have no UIDNEXT and if in doubt, return true.
+                true
+            };
+
+            return Ok(new_emails);
+        }
+
+        // UIDVALIDITY is modified, reset highest seen MODSEQ.
+        set_modseq(context, folder, 0).await?;
+
+        // ==============  uid_validity has changed or is being set the first time.  ==============
+
+        let new_uid_next = new_uid_next.unwrap_or_default();
+        set_uid_next(context, folder, new_uid_next).await?;
+        set_uidvalidity(context, folder, new_uid_validity).await?;
+
+        // Collect garbage entries in `imap` table.
+        context
+            .sql
+            .execute(
+                "DELETE FROM imap WHERE folder=? AND uidvalidity!=?",
+                (&folder, new_uid_validity),
+            )
+            .await?;
+
+        if old_uid_validity != 0 || old_uid_next != 0 {
+            context.schedule_resync().await?;
+        }
+        info!(
+            context,
+            "uid/validity change folder {}: new {}/{} previous {}/{}.",
+            folder,
+            new_uid_next,
+            new_uid_validity,
+            old_uid_next,
+            old_uid_validity,
+        );
+        Ok(false)
+    }
 }
 
 #[derive(PartialEq, Debug, Copy, Clone, Eq)]
-pub(super) enum NewlySelected {
+pub(crate) enum NewlySelected {
     /// The folder was newly selected during this call to select_folder().
     Yes,
     /// No SELECT command was run because the folder already was selected

src/imap/session.rs

@@ -1,11 +1,32 @@
+use std::cmp;
+use std::collections::BTreeMap;
 use std::ops::{Deref, DerefMut};
 
+use anyhow::{Context as _, Result};
 use async_imap::types::Mailbox;
 use async_imap::Session as ImapSession;
+use futures::TryStreamExt;
 
+use crate::constants::DC_FETCH_EXISTING_MSGS_COUNT;
 use crate::imap::capabilities::Capabilities;
 use crate::net::session::SessionStream;
 
+/// Prefetch:
+/// - Message-ID to check if we already have the message.
+/// - In-Reply-To and References to check if message is a reply to chat message.
+/// - Chat-Version to check if a message is a chat message
+/// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
+///   not necessarily sent by Delta Chat.
+const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
+                              MESSAGE-ID \
+                              DATE \
+                              X-MICROSOFT-ORIGINAL-MESSAGE-ID \
+                              FROM \
+                              IN-REPLY-TO REFERENCES \
+                              CHAT-VERSION \
+                              AUTOCRYPT-SETUP-MESSAGE\
+                              )])";
+
 #[derive(Debug)]
 pub(crate) struct Session {
     pub(super) inner: ImapSession<Box<dyn SessionStream>>,
@@ -64,4 +85,79 @@ impl Session {
     pub fn can_condstore(&self) -> bool {
         self.capabilities.can_condstore
     }
+
+    pub fn can_metadata(&self) -> bool {
+        self.capabilities.can_metadata
+    }
+
+    pub fn can_push(&self) -> bool {
+        self.capabilities.can_push
+    }
+
+    /// Returns the names of all folders on the IMAP server.
+    pub async fn list_folders(&mut self) -> Result<Vec<async_imap::types::Name>> {
+        let list = self.list(Some(""), Some("*")).await?.try_collect().await?;
+        Ok(list)
+    }
+
+    /// Prefetch all messages greater than or equal to `uid_next`. Returns a list of fetch results
+    /// in the order of ascending delivery time to the server (INTERNALDATE).
+    pub(crate) async fn prefetch(
+        &mut self,
+        uid_next: u32,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        // fetch messages with larger UID than the last one seen
+        let set = format!("{uid_next}:*");
+        let mut list = self
+            .uid_fetch(set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                // If the mailbox is not empty, results always include
+                // at least one UID, even if last_seen_uid+1 is past
+                // the last UID in the mailbox.  It happens because
+                // uid:* is interpreted the same way as *:uid.
+                // See <https://tools.ietf.org/html/rfc3501#page-61> for
+                // standard reference. Therefore, sometimes we receive
+                // already seen messages and have to filter them out.
+                if msg_uid >= uid_next {
+                    msgs.insert((msg.internal_date(), msg_uid), msg);
+                }
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
+
+    /// Like prefetch(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
+    pub(crate) async fn prefetch_existing_msgs(
+        &mut self,
+    ) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
+        let exists: i64 = {
+            let mailbox = self.selected_mailbox.as_ref().context("no mailbox")?;
+            mailbox.exists.into()
+        };
+
+        // Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
+        // Sequence numbers are sequential. If there are 1000 messages in the inbox,
+        // we can fetch the sequence numbers 900-1000 and get the last 100 messages.
+        let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT + 1);
+        let set = format!("{first}:{exists}");
+        let mut list = self
+            .fetch(&set, PREFETCH_FLAGS)
+            .await
+            .context("IMAP Could not fetch")?;
+
+        let mut msgs = BTreeMap::new();
+        while let Some(msg) = list.try_next().await? {
+            if let Some(msg_uid) = msg.uid {
+                msgs.insert((msg.internal_date(), msg_uid), msg);
+            }
+        }
+
+        Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
+    }
 }

src/imex.rs

@@ -389,11 +389,11 @@ async fn imex_inner(
 
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
-        if e2ee::ensure_secret_key_exists(context).await.is_err() {
-            bail!("Cannot create private key or private key not available.");
-        } else {
-            create_folder(context, &path).await?;
-        }
+        e2ee::ensure_secret_key_exists(context)
+            .await
+            .context("Cannot create private key or private key not available")?;
+
+        create_folder(context, &path).await?;
     }
 
     match what {
@@ -534,7 +534,7 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
     let _d1 = DeleteOnDrop(temp_db_path.clone());
     let _d2 = DeleteOnDrop(temp_path.clone());
 
-    export_database(context, &temp_db_path, passphrase)
+    export_database(context, &temp_db_path, passphrase, now)
         .await
         .context("could not export database")?;
 
@@ -770,19 +770,27 @@ where
 /// overwritten.
 ///
 /// This also verifies that IO is not running during the export.
-async fn export_database(context: &Context, dest: &Path, passphrase: String) -> Result<()> {
+async fn export_database(
+    context: &Context,
+    dest: &Path,
+    passphrase: String,
+    timestamp: i64,
+) -> Result<()> {
     ensure!(
         !context.scheduler.is_running().await,
         "cannot export backup, IO is running"
     );
-    let now = time().try_into().context("32-bit UNIX time overflow")?;
+    let timestamp = timestamp.try_into().context("32-bit UNIX time overflow")?;
 
     // TODO: Maybe introduce camino crate for UTF-8 paths where we need them.
     let dest = dest
         .to_str()
         .with_context(|| format!("path {} is not valid unicode", dest.display()))?;
 
-    context.sql.set_raw_config_int("backup_time", now).await?;
+    context
+        .sql
+        .set_raw_config_int("backup_time", timestamp)
+        .await?;
     sql::housekeeping(context).await.log_err(context).ok();
     context
         .sql
@@ -819,7 +827,7 @@ mod tests {
     use crate::key;
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
     use crate::stock_str::StockMessage;
-    use crate::test_utils::{alice_keypair, TestContext};
+    use crate::test_utils::{alice_keypair, TestContext, TestContextManager};
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_setup_file() {
@@ -1087,6 +1095,10 @@ mod tests {
     const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
     const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
 
+    // Autocrypt Setup Message payload "encrypted" with plaintext algorithm.
+    const S_PLAINTEXT_SETUPFILE: &str =
+        include_str!("../test-data/message/plaintext-autocrypt-setup.txt");
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_split_and_decrypt() {
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
@@ -1110,6 +1122,23 @@ mod tests {
         assert!(headers.get(HEADER_SETUPCODE).is_none());
     }
 
+    /// Tests that Autocrypt Setup Message encrypted with "plaintext" algorithm cannot be
+    /// decrypted.
+    ///
+    /// According to <https://datatracker.ietf.org/doc/html/rfc4880#section-13.4>
+    /// "Implementations MUST NOT use plaintext in Symmetrically Encrypted Data packets".
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decrypt_plaintext_autocrypt_setup_message() {
+        let setup_file = S_PLAINTEXT_SETUPFILE.to_string();
+        let incorrect_setupcode = "0000-0000-0000-0000-0000-0000-0000-0000-0000";
+        assert!(decrypt_setup_file(
+            incorrect_setupcode,
+            std::io::Cursor::new(setup_file.as_bytes()),
+        )
+        .await
+        .is_err());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_key_transfer() -> Result<()> {
         let alice = TestContext::new_alice().await;
@@ -1125,6 +1154,7 @@ mod tests {
         alice2.configure_addr("alice@example.org").await;
         alice2.recv_msg(&sent).await;
         let msg = alice2.get_last_msg().await;
+        assert!(msg.is_setupmessage());
 
         // Send a message that cannot be decrypted because the keys are
         // not synchronized yet.
@@ -1142,4 +1172,25 @@ mod tests {
 
         Ok(())
     }
+
+    /// Tests that Autocrypt Setup Messages is only clickable if it is self-sent.
+    /// This prevents Bob from tricking Alice into changing the key
+    /// by sending her an Autocrypt Setup Message as long as Alice's server
+    /// does not allow to forge the `From:` header.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_key_transfer_non_self_sent() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        let _setup_code = initiate_key_transfer(&alice).await?;
+
+        // Get Autocrypt Setup Message.
+        let sent = alice.pop_sent_msg().await;
+
+        let rcvd = bob.recv_msg(&sent).await;
+        assert!(!rcvd.is_setupmessage());
+
+        Ok(())
+    }
 }

src/imex/transfer.rs

@@ -52,6 +52,7 @@ use crate::context::Context;
 use crate::message::{Message, Viewtype};
 use crate::qr::{self, Qr};
 use crate::stock_str::backup_transfer_msg_body;
+use crate::tools::time;
 use crate::{e2ee, EventType};
 
 use super::{export_database, DBFILE_BACKUP_NAME};
@@ -158,7 +159,7 @@ impl BackupProvider {
         // Generate the token up front: we also use it to encrypt the database.
         let token = AuthToken::generate();
         context.emit_event(SendProgress::Started.into());
-        export_database(context, dbfile, token.to_string())
+        export_database(context, dbfile, token.to_string(), time())
             .await
             .context("Database export failed")?;
         context.emit_event(SendProgress::DatabaseExported.into());

src/key.rs

@@ -18,7 +18,7 @@ use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::log::LogExt;
 use crate::pgp::KeyPair;
-use crate::tools::EmailAddress;
+use crate::tools::{self, time_elapsed, EmailAddress};
 
 /// Convenience trait for working with keys.
 ///
@@ -79,7 +79,7 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
 }
 
 pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPublicKey> {
-    match context
+    let public_key = context
         .sql
         .query_row_optional(
             "SELECT public_key
@@ -91,8 +91,8 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
                 Ok(bytes)
             },
         )
-        .await?
-    {
+        .await?;
+    match public_key {
         Some(bytes) => SignedPublicKey::from_slice(&bytes),
         None => {
             let keypair = generate_keypair(context).await?;
@@ -101,8 +101,27 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
     }
 }
 
+/// Returns our own public keyring.
+pub(crate) async fn load_self_public_keyring(context: &Context) -> Result<Vec<SignedPublicKey>> {
+    let keys = context
+        .sql
+        .query_map(
+            r#"SELECT public_key
+               FROM keypairs
+               ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
+            (),
+            |row| row.get::<_, Vec<u8>>(0),
+            |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
+        .await?
+        .into_iter()
+        .filter_map(|bytes| SignedPublicKey::from_slice(&bytes).log_err(context).ok())
+        .collect();
+    Ok(keys)
+}
+
 pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecretKey> {
-    match context
+    let private_key = context
         .sql
         .query_row_optional(
             "SELECT private_key
@@ -114,8 +133,8 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
                 Ok(bytes)
             },
         )
-        .await?
-    {
+        .await?;
+    match private_key {
         Some(bytes) => SignedSecretKey::from_slice(&bytes),
         None => {
             let keypair = generate_keypair(context).await?;
@@ -204,7 +223,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     match load_keypair(context, &addr).await? {
         Some(key_pair) => Ok(key_pair),
         None => {
-            let start = std::time::SystemTime::now();
+            let start = tools::Time::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await?)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
@@ -216,7 +235,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().unwrap_or_default().as_secs()
+                time_elapsed(&start).as_secs(),
             );
             Ok(keypair)
         }
@@ -255,7 +274,7 @@ pub(crate) async fn load_keypair(
 
 /// Use of a key pair for encryption or decryption.
 ///
-/// This is used by [store_self_keypair] to know what kind of key is
+/// This is used by `store_self_keypair` to know what kind of key is
 /// being saved.
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub enum KeyPairUse {
@@ -277,7 +296,7 @@ pub enum KeyPairUse {
 /// same key again overwrites it.
 ///
 /// [Config::ConfiguredAddr]: crate::config::Config::ConfiguredAddr
-pub async fn store_self_keypair(
+pub(crate) async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,
@@ -294,11 +313,17 @@ pub async fn store_self_keypair(
                 KeyPairUse::ReadOnly => false,
             };
 
+            // `addr` and `is_default` written for compatibility with older versions,
+            // until new cores are rolled out everywhere.
+            // otherwise "add second device" or "backup" may break.
+            // moreover, this allows downgrades to the previous version.
+            // writing of `addr` and `is_default` can be removed ~ 2024-08
+            let addr = keypair.addr.to_string();
             transaction
                 .execute(
-                    "INSERT OR REPLACE INTO keypairs (public_key, private_key)
-                     VALUES (?,?)",
-                    (&public_key, &secret_key),
+                    "INSERT OR REPLACE INTO keypairs (public_key, private_key, addr, is_default)
+                     VALUES (?,?,?,?)",
+                    (&public_key, &secret_key, addr, is_default),
                 )
                 .context("Failed to insert keypair")?;
 

src/lib.rs

@@ -98,6 +98,7 @@ mod color;
 pub mod html;
 pub mod net;
 pub mod plaintext;
+mod push;
 pub mod summary;
 
 mod debug_logging;
@@ -105,7 +106,6 @@ pub mod receive_imf;
 pub mod tools;
 
 pub mod accounts;
-pub mod peer_channels;
 pub mod reaction;
 
 /// If set IMAP/incoming and SMTP/outgoing MIME messages will be printed.

src/location.rs

@@ -139,8 +139,9 @@ impl Kml {
                 match chrono::NaiveDateTime::parse_from_str(&val, "%Y-%m-%dT%H:%M:%SZ") {
                     Ok(res) => {
                         self.curr.timestamp = res.timestamp();
-                        if self.curr.timestamp > time() {
-                            self.curr.timestamp = time();
+                        let now = time();
+                        if self.curr.timestamp > now {
+                            self.curr.timestamp = now;
                         }
                     }
                     Err(_err) => {
@@ -333,12 +334,13 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         return Ok(true);
     }
     let mut continue_streaming = false;
+    let now = time();
 
     let chats = context
         .sql
         .query_map(
             "SELECT id FROM chats WHERE locations_send_until>?;",
-            (time(),),
+            (now,),
             |row| row.get::<_, i32>(0),
             |chats| {
                 chats
@@ -356,7 +358,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                     latitude,
                     longitude,
                     accuracy,
-                    time(),
+                    now,
                     chat_id,
                     ContactId::SELF,
                 )).await.context("Failed to store location")?;

src/message.rs

@@ -757,7 +757,7 @@ impl Message {
         self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
-    /// Returns true if message is Auto-Submitted.
+    /// Returns true if message is auto-generated.
     pub fn is_bot(&self) -> bool {
         self.param.get_bool(Param::Bot).unwrap_or_default()
     }
@@ -1130,7 +1130,7 @@ impl Message {
     /// `References` header is not taken into account.
     pub async fn parent(&self, context: &Context) -> Result<Option<Message>> {
         if let Some(in_reply_to) = &self.in_reply_to {
-            if let Some(msg_id) = rfc724_mid_exists(context, in_reply_to).await? {
+            if let Some((msg_id, _ts_sent)) = rfc724_mid_exists(context, in_reply_to).await? {
                 let msg = Message::load_from_db(context, msg_id).await?;
                 return if msg.chat_id.is_trash() {
                     // If message is already moved to trash chat, pretend it does not exist.
@@ -1522,7 +1522,9 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
 
     if !msg_ids.is_empty() {
         // Run housekeeping to delete unused blobs.
-        context.set_config(Config::LastHousekeeping, None).await?;
+        context
+            .set_config_internal(Config::LastHousekeeping, None)
+            .await?;
     }
 
     // Interrupt Inbox loop to start message deletion and run housekeeping.
@@ -1550,7 +1552,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
     let old_last_msg_id = MsgId::new(context.get_config_u32(Config::LastMsgId).await?);
     let last_msg_id = msg_ids.iter().fold(&old_last_msg_id, std::cmp::max);
     context
-        .set_config_u32(Config::LastMsgId, last_msg_id.to_u32())
+        .set_config_internal(Config::LastMsgId, Some(&last_msg_id.to_u32().to_string()))
         .await?;
 
     let msgs = context
@@ -1816,18 +1818,23 @@ pub async fn estimate_deletion_cnt(
     Ok(cnt)
 }
 
+/// See [`rfc724_mid_exists_and()`].
 pub(crate) async fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<Option<MsgId>> {
+) -> Result<Option<(MsgId, i64)>> {
     rfc724_mid_exists_and(context, rfc724_mid, "1").await
 }
 
+/// Returns [MsgId] and "sent" timestamp of the message with given `rfc724_mid` (Message-ID header)
+/// if it exists in the db.
+///
+/// @param cond SQL subexpression for filtering messages.
 pub(crate) async fn rfc724_mid_exists_and(
     context: &Context,
     rfc724_mid: &str,
     cond: &str,
-) -> Result<Option<MsgId>> {
+) -> Result<Option<(MsgId, i64)>> {
     let rfc724_mid = rfc724_mid.trim_start_matches('<').trim_end_matches('>');
     if rfc724_mid.is_empty() {
         warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
@@ -1837,12 +1844,13 @@ pub(crate) async fn rfc724_mid_exists_and(
     let res = context
         .sql
         .query_row_optional(
-            &("SELECT id FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
+            &("SELECT id, timestamp_sent FROM msgs WHERE rfc724_mid=? AND ".to_string() + cond),
             (rfc724_mid,),
             |row| {
                 let msg_id: MsgId = row.get(0)?;
+                let timestamp_sent: i64 = row.get(1)?;
 
-                Ok(msg_id)
+                Ok((msg_id, timestamp_sent))
             },
         )
         .await?;
@@ -1858,7 +1866,7 @@ pub(crate) async fn get_latest_by_rfc724_mids(
     mids: &[String],
 ) -> Result<Option<Message>> {
     for id in mids.iter().rev() {
-        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
+        if let Some((msg_id, _)) = rfc724_mid_exists(context, id).await? {
             let msg = Message::load_from_db(context, msg_id).await?;
             if msg.chat_id != DC_CHAT_ID_TRASH {
                 return Ok(Some(msg));

src/mimefactory.rs

@@ -18,7 +18,6 @@ use crate::contact::Contact;
 use crate::context::Context;
 use crate::e2ee::EncryptHelper;
 use crate::ephemeral::Timer as EphemeralTimer;
-use crate::headerdef::HeaderDef;
 use crate::html::new_html_mimepart;
 use crate::location;
 use crate::message::{self, Message, MsgId, Viewtype};
@@ -339,18 +338,11 @@ impl<'a> MimeFactory<'a> {
     fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.is_protected() {
-                    false
-                } else if chat.typ == Chattype::Broadcast {
-                    // encryption may disclose recipients;
-                    // this is probably a worse issue than not opportunistically (!) encrypting
-                    true
-                } else {
-                    self.msg
-                        .param
-                        .get_bool(Param::ForcePlaintext)
-                        .unwrap_or_default()
-                }
+                self.msg
+                    .param
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    || chat.typ == Chattype::Broadcast
             }
             Loaded::Mdn { .. } => false,
         }
@@ -367,20 +359,25 @@ impl<'a> MimeFactory<'a> {
         }
     }
 
-    async fn should_do_gossip(&self, context: &Context) -> Result<bool> {
+    async fn should_do_gossip(&self, context: &Context, multiple_recipients: bool) -> Result<bool> {
         match &self.loaded {
             Loaded::Message { chat } => {
-                // beside key- and member-changes, force a periodic re-gossip.
-                let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
-                let gossip_period = context.get_config_i64(Config::GossipPeriod).await?;
-                if time() >= gossiped_timestamp + gossip_period {
+                let cmd = self.msg.param.get_cmd();
+                if cmd == SystemMessage::MemberAddedToGroup
+                    || cmd == SystemMessage::SecurejoinMessage
+                {
                     Ok(true)
+                } else if multiple_recipients {
+                    // beside key- and member-changes, force a periodic re-gossip.
+                    let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
+                    let gossip_period = context.get_config_i64(Config::GossipPeriod).await?;
+                    if time() >= gossiped_timestamp + gossip_period {
+                        Ok(true)
+                    } else {
+                        Ok(false)
+                    }
                 } else {
-                    let cmd = self.msg.param.get_cmd();
-                    // Do gossip in all Securejoin messages not to complicate the code. There's no
-                    // need in gossips in "vg-auth-required" messages f.e., but let them be.
-                    Ok(cmd == SystemMessage::MemberAddedToGroup
-                        || cmd == SystemMessage::SecurejoinMessage)
+                    Ok(false)
                 }
             }
             Loaded::Mdn { .. } => Ok(false),
@@ -559,7 +556,7 @@ impl<'a> MimeFactory<'a> {
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
-            Loaded::Mdn { .. } => create_outgoing_rfc724_mid(None, &self.from_addr),
+            Loaded::Mdn { .. } => create_outgoing_rfc724_mid(&self.from_addr),
         };
         let rfc724_mid_headervalue = render_rfc724_mid(&rfc724_mid);
         let rfc724_mid_header = Header::new("Message-ID".into(), rfc724_mid_headervalue);
@@ -706,9 +703,9 @@ impl<'a> MimeFactory<'a> {
                 .fold(message, |message, header| message.header(header));
 
             // Add gossip headers in chats with multiple recipients
-            if (peerstates.len() > 1 || context.get_config_bool(Config::BccSelf).await?)
-                && self.should_do_gossip(context).await?
-            {
+            let multiple_recipients =
+                peerstates.len() > 1 || context.get_config_bool(Config::BccSelf).await?;
+            if self.should_do_gossip(context, multiple_recipients).await? {
                 for peerstate in peerstates.iter().filter_map(|(state, _)| state.as_ref()) {
                     if let Some(header) = peerstate.render_gossip_header(verified) {
                         message = message.header(Header::new("Autocrypt-Gossip".into(), header));
@@ -742,8 +739,12 @@ impl<'a> MimeFactory<'a> {
                 );
             }
 
+            // Disable compression for SecureJoin to ensure
+            // there are no compression side channels
+            // leaking information about the tokens.
+            let compress = self.msg.param.get_cmd() != SystemMessage::SecurejoinMessage;
             let encrypted = encrypt_helper
-                .encrypt(context, verified, message, peerstates)
+                .encrypt(context, verified, message, peerstates, compress)
                 .await?;
 
             outer_message
@@ -941,7 +942,6 @@ impl<'a> MimeFactory<'a> {
         };
         let command = self.msg.param.get_cmd();
         let mut placeholdertext = None;
-        let mut meta_part = None;
 
         let send_verified_headers = match chat.typ {
             Chattype::Single => true,
@@ -1127,17 +1127,13 @@ impl<'a> MimeFactory<'a> {
 
         if let Some(grpimage) = grpimage {
             info!(context, "setting group image '{}'", grpimage);
-            let mut meta = Message {
-                viewtype: Viewtype::Image,
-                ..Default::default()
-            };
-            meta.param.set(Param::File, grpimage);
-
-            let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image").await?;
-            meta_part = Some(mail);
-            headers
-                .protected
-                .push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            let avatar = build_avatar_file(context, grpimage)
+                .await
+                .context("Cannot attach group image")?;
+            headers.hidden.push(Header::new(
+                "Chat-Group-Avatar".into(),
+                format!("base64:{avatar}"),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
@@ -1261,10 +1257,6 @@ impl<'a> MimeFactory<'a> {
             parts.push(file_part);
         }
 
-        if let Some(meta_part) = meta_part {
-            parts.push(meta_part);
-        }
-
         if let Some(msg_kml_part) = self.get_message_kml_part() {
             parts.push(msg_kml_part);
         }
@@ -1275,7 +1267,6 @@ impl<'a> MimeFactory<'a> {
             }
         }
 
-        println!("hiiiii");
         // we do not piggyback sync-files to other self-sent-messages
         // to not risk files becoming too larger and being skipped by download-on-demand.
         if command == SystemMessage::MultiDeviceSync && self.is_e2ee_guaranteed() {
@@ -1285,16 +1276,6 @@ impl<'a> MimeFactory<'a> {
             self.sync_ids_to_delete = Some(ids.to_string());
         } else if command == SystemMessage::WebxdcStatusUpdate {
             let json = self.msg.param.get(Param::Arg).unwrap_or_default();
-            if json.find("gossip_topic").is_some() {
-                if let Some(ref endpoint) = *context.endpoint.lock().await {
-                    // Add iroh NodeAddr to headers so peers can connect to us.
-                    let node_addr = endpoint.my_addr().await.unwrap();
-                    headers.protected.push(Header::new(
-                        HeaderDef::IrohPublicGossip.get_headername().to_string(),
-                        serde_json::to_string(&node_addr)?,
-                    ));
-                }
-            }
             parts.push(context.build_status_update_part(json));
         } else if self.msg.viewtype == Viewtype::Webxdc {
             if let Some(json) = context
@@ -1307,7 +1288,7 @@ impl<'a> MimeFactory<'a> {
 
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar).await? {
-                Some(path) => match build_selfavatar_file(context, &path).await {
+                Some(path) => match build_avatar_file(context, &path).await {
                     Ok(avatar) => headers.hidden.push(Header::new(
                         "Chat-User-Avatar".into(),
                         format!("base64:{avatar}"),
@@ -1516,8 +1497,11 @@ async fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-async fn build_selfavatar_file(context: &Context, path: &str) -> Result<String> {
-    let blob = BlobObject::from_path(context, path.as_ref())?;
+async fn build_avatar_file(context: &Context, path: &str) -> Result<String> {
+    let blob = match path.starts_with("$BLOBDIR/") {
+        true => BlobObject::from_name(context, path.to_string())?,
+        false => BlobObject::from_path(context, path.as_ref())?,
+    };
     let body = fs::read(blob.to_abs_path()).await?;
     let encoded_body = wrapped_base64_encode(&body);
     Ok(encoded_body)

src/mimeparser.rs

@@ -8,7 +8,6 @@ use std::pin::Pin;
 use std::str;
 
 use anyhow::{bail, Context as _, Result};
-use base64::Engine as _;
 use deltachat_derive::{FromSql, ToSql};
 use format_flowed::unformat_flowed;
 use lettre_email::mime::{self, Mime};
@@ -69,6 +68,8 @@ pub(crate) struct MimeMessage {
     /// Whether the From address was repeated in the signed part
     /// (and we know that the signer intended to send from this address)
     pub from_is_signed: bool,
+    /// Whether the message is incoming or outgoing (self-sent).
+    pub incoming: bool,
     /// The List-Post address is only set for mailing lists. Users can send
     /// messages to this address to post them to the list.
     pub list_post: Option<String>,
@@ -82,9 +83,10 @@ pub(crate) struct MimeMessage {
     /// If a message is not encrypted or the signature is not valid,
     /// this set is empty.
     pub signatures: HashSet<Fingerprint>,
-    /// The set of mail recipient addresses for which gossip headers were applied, regardless of
-    /// whether they modified any peerstates.
-    pub gossiped_addr: HashSet<String>,
+    /// The mail recipient addresses for which gossip headers were applied
+    /// and their respective gossiped keys,
+    /// regardless of whether they modified any peerstates.
+    pub gossiped_keys: HashMap<String, SignedPublicKey>,
 
     /// True if the message is a forwarded message.
     pub is_forwarded: bool,
@@ -117,8 +119,8 @@ pub(crate) struct MimeMessage {
     /// Hop info for debugging.
     pub(crate) hop_info: String,
 
-    /// Whether the contact sending this should be marked as bot.
-    pub(crate) is_bot: bool,
+    /// Whether the contact sending this should be marked as bot or non-bot.
+    pub(crate) is_bot: Option<bool>,
 
     /// When the message was received, in secs since epoch.
     pub(crate) timestamp_rcvd: i64,
@@ -207,11 +209,12 @@ impl MimeMessage {
     ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
-        let message_time = mail
+        let timestamp_rcvd = smeared_time(context);
+        let timestamp_sent = mail
             .headers
             .get_header_value(HeaderDef::Date)
             .and_then(|v| mailparse::dateparse(&v).ok())
-            .unwrap_or_default();
+            .map_or(timestamp_rcvd, |value| min(value, timestamp_rcvd + 60));
         let mut hop_info = parse_receive_headers(&mail.get_headers());
 
         let mut headers = Default::default();
@@ -262,14 +265,28 @@ impl MimeMessage {
                 for field in &part.headers {
                     let key = field.get_key().to_lowercase();
 
-                    // For now only Chat-User-Avatar can be hidden.
-                    if !headers.contains_key(&key) && key == "chat-user-avatar" {
+                    // For now only avatar headers can be hidden.
+                    if !headers.contains_key(&key)
+                        && (key == "chat-user-avatar" || key == "chat-group-avatar")
+                    {
                         headers.insert(key.to_string(), field.get_value());
                     }
                 }
             }
         }
 
+        // Overwrite Message-ID with X-Microsoft-Original-Message-ID.
+        // However if we later find Message-ID in the protected part,
+        // it will overwrite both.
+        if let Some(microsoft_message_id) =
+            headers.remove(HeaderDef::XMicrosoftOriginalMessageId.get_headername())
+        {
+            headers.insert(
+                HeaderDef::MessageId.get_headername().to_string(),
+                microsoft_message_id,
+            );
+        }
+
         // Remove headers that are allowed _only_ in the encrypted+signed part. It's ok to leave
         // them in signed-only emails, but has no value currently.
         Self::remove_secured_headers(&mut headers);
@@ -278,16 +295,17 @@ impl MimeMessage {
         let private_keyring = load_self_secret_keyring(context).await?;
 
         let mut decryption_info =
-            prepare_decryption(context, &mail, &from.addr, message_time).await?;
+            prepare_decryption(context, &mail, &from.addr, timestamp_sent).await?;
 
         // Memory location for a possible decrypted message.
         let mut mail_raw = Vec::new();
-        let mut gossiped_addr = Default::default();
+        let mut gossiped_keys = Default::default();
         let mut from_is_signed = false;
         hop_info += "\n\n";
         hop_info += &decryption_info.dkim_results.to_string();
 
-        let public_keyring = keyring_from_peerstate(decryption_info.peerstate.as_ref());
+        let public_keyring =
+            keyring_from_peerstate(context, decryption_info.peerstate.as_ref()).await?;
         let (mail, mut signatures, encrypted) = match tokio::task::block_in_place(|| {
             try_decrypt(&mail, &private_keyring, &public_keyring)
         }) {
@@ -322,17 +340,28 @@ impl MimeMessage {
                 // but only if the mail was correctly signed. Probably it's ok to not require
                 // encryption here, but let's follow the standard.
                 let gossip_headers = mail.headers.get_all_values("Autocrypt-Gossip");
-                gossiped_addr = update_gossip_peerstates(
+                gossiped_keys = update_gossip_peerstates(
                     context,
-                    message_time,
+                    timestamp_sent,
                     &from.addr,
                     &recipients,
                     gossip_headers,
                 )
                 .await?;
-                // Remove unsigned subject from messages displayed with padlock.
-                // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
-                headers.remove("subject");
+                // Remove unsigned opportunistically protected headers from messages considered
+                // Autocrypt-encrypted / displayed with padlock.
+                // For "Subject" see <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
+                for h in [
+                    HeaderDef::Subject,
+                    HeaderDef::ChatGroupId,
+                    HeaderDef::ChatGroupName,
+                    HeaderDef::ChatGroupNameChanged,
+                    HeaderDef::ChatGroupAvatar,
+                    HeaderDef::ChatGroupMemberRemoved,
+                    HeaderDef::ChatGroupMemberAdded,
+                ] {
+                    headers.remove(h.get_headername());
+                }
             }
 
             // let known protected headers from the decrypted
@@ -360,13 +389,20 @@ impl MimeMessage {
                     // signed part, but it doesn't match the outer one.
                     // This _might_ be because the sender's mail server
                     // replaced the sending address, e.g. in a mailing list.
-                    // Or it's because someone is doing some replay attack
-                    // - OTOH, I can't come up with an attack scenario
-                    // where this would be useful.
+                    // Or it's because someone is doing some replay attack.
+                    // Resending encrypted messages via mailing lists
+                    // without reencrypting is not useful anyway,
+                    // so we return an error below.
                     warn!(
                         context,
                         "From header in signed part doesn't match the outer one",
                     );
+
+                    // Return an error from the parser.
+                    // This will result in creating a tombstone
+                    // and no further message processing
+                    // as if the MIME structure is broken.
+                    bail!("From header is forged");
                 }
             }
         }
@@ -375,12 +411,12 @@ impl MimeMessage {
 
             // If it is not a read receipt, degrade encryption.
             if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
-                if message_time > peerstate.last_seen_autocrypt
+                if timestamp_sent > peerstate.last_seen_autocrypt
                     && mail.ctype.mimetype != "multipart/report"
                 // Disallowing keychanges is disabled for now:
                 // && decryption_info.dkim_results.allow_keychange
                 {
-                    peerstate.degrade_encryption(message_time);
+                    peerstate.degrade_encryption(timestamp_sent);
                 }
             }
         }
@@ -394,15 +430,7 @@ impl MimeMessage {
             }
         }
 
-        // Auto-submitted is also set by holiday-notices so we also check `chat-version`
-        let is_bot = headers.contains_key("auto-submitted") && headers.contains_key("chat-version");
-
-        let timestamp_rcvd = smeared_time(context);
-        let timestamp_sent = headers
-            .get(HeaderDef::Date.get_headername())
-            .and_then(|value| mailparse::dateparse(value).ok())
-            .map_or(timestamp_rcvd, |value| min(value, timestamp_rcvd + 60));
-
+        let incoming = !context.is_self_addr(&from.addr).await?;
         let mut parser = MimeMessage {
             parts: Vec::new(),
             headers,
@@ -410,13 +438,14 @@ impl MimeMessage {
             list_post,
             from,
             from_is_signed,
+            incoming,
             chat_disposition_notification_to,
             decryption_info,
             decrypting_failed: mail.is_err(),
 
             // only non-empty if it was a valid autocrypt message
             signatures,
-            gossiped_addr,
+            gossiped_keys,
             is_forwarded: false,
             mdn_reports: Vec::new(),
             is_system_message: SystemMessage::Unknown,
@@ -431,7 +460,7 @@ impl MimeMessage {
             is_mime_modified: false,
             decoded_data: Vec::new(),
             hop_info,
-            is_bot,
+            is_bot: None,
             timestamp_rcvd,
             timestamp_sent,
         };
@@ -464,6 +493,13 @@ impl MimeMessage {
             },
         };
 
+        if parser.mdn_reports.is_empty() {
+            // "Auto-Submitted" is also set by holiday-notices so we also check "chat-version".
+            let is_bot = parser.headers.get("auto-submitted")
+                == Some(&"auto-generated".to_string())
+                && parser.headers.contains_key("chat-version");
+            parser.is_bot = Some(is_bot);
+        }
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
@@ -485,7 +521,7 @@ impl MimeMessage {
 
     /// Parses system messages.
     fn parse_system_message_headers(&mut self, context: &Context) {
-        if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() {
+        if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() && !self.incoming {
             self.parts.retain(|part| {
                 part.mimetype.is_none()
                     || part.mimetype.as_ref().unwrap().as_ref() == MIME_AC_SETUP_FILE
@@ -702,7 +738,7 @@ impl MimeMessage {
             self.do_add_single_part(part);
         }
 
-        if self.headers.contains_key("auto-submitted") {
+        if self.is_bot == Some(true) {
             for part in &mut self.parts {
                 part.param.set(Param::Bot, "1");
             }
@@ -718,37 +754,20 @@ impl MimeMessage {
     ) -> Option<AvatarAction> {
         if header_value == "0" {
             Some(AvatarAction::Delete)
-        } else if let Some(avatar) = header_value
+        } else if let Some(base64) = header_value
             .split_ascii_whitespace()
             .collect::<String>()
             .strip_prefix("base64:")
-            .map(|x| base64::engine::general_purpose::STANDARD.decode(x))
         {
-            // Avatar sent directly in the header as base64.
-            if let Ok(decoded_data) = avatar {
-                let extension = if let Ok(format) = image::guess_format(&decoded_data) {
-                    if let Some(ext) = format.extensions_str().first() {
-                        format!(".{ext}")
-                    } else {
-                        String::new()
-                    }
-                } else {
-                    String::new()
-                };
-                match BlobObject::create(context, &format!("avatar{extension}"), &decoded_data)
-                    .await
-                {
-                    Ok(blob) => Some(AvatarAction::Change(blob.as_name().to_string())),
-                    Err(err) => {
-                        warn!(
-                            context,
-                            "Could not save decoded avatar to blob file: {:#}", err
-                        );
-                        None
-                    }
+            match BlobObject::store_from_base64(context, base64, "avatar").await {
+                Ok(path) => Some(AvatarAction::Change(path)),
+                Err(err) => {
+                    warn!(
+                        context,
+                        "Could not decode and save avatar to blob file: {:#}", err,
+                    );
+                    None
                 }
-            } else {
-                None
             }
         } else {
             // Avatar sent in attachment, as previous versions of Delta Chat did.
@@ -974,10 +993,13 @@ impl MimeMessage {
                             }
                         }
                         Some(_) => {
-                            if let Some(first) = mail.subparts.first() {
-                                any_part_added = self
-                                    .parse_mime_recursive(context, first, is_related)
-                                    .await?;
+                            for cur_data in &mail.subparts {
+                                if self
+                                    .parse_mime_recursive(context, cur_data, is_related)
+                                    .await?
+                                {
+                                    any_part_added = true;
+                                }
                             }
                         }
                     }
@@ -1388,14 +1410,22 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_rfc724_mid(&self) -> Option<String> {
-        self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
-            .or_else(|| self.get_header(HeaderDef::MessageId))
+        self.get_header(HeaderDef::MessageId)
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
     fn remove_secured_headers(headers: &mut HashMap<String, String>) {
         headers.remove("secure-join-fingerprint");
+        headers.remove("secure-join-auth");
         headers.remove("chat-verified");
+        headers.remove("autocrypt-gossip");
+
+        // Secure-Join is secured unless it is an initial "vc-request"/"vg-request".
+        if let Some(secure_join) = headers.remove("secure-join") {
+            if secure_join == "vc-request" || secure_join == "vg-request" {
+                headers.insert("secure-join".to_string(), secure_join);
+            }
+        }
     }
 
     fn merge_headers(
@@ -1721,9 +1751,9 @@ async fn update_gossip_peerstates(
     from: &str,
     recipients: &[SingleInfo],
     gossip_headers: Vec<String>,
-) -> Result<HashSet<String>> {
+) -> Result<HashMap<String, SignedPublicKey>> {
     // XXX split the parsing from the modification part
-    let mut gossiped_addr: HashSet<String> = Default::default();
+    let mut gossiped_keys: HashMap<String, SignedPublicKey> = Default::default();
 
     for value in &gossip_headers {
         let header = match value.parse::<Aheader>() {
@@ -1767,10 +1797,10 @@ async fn update_gossip_peerstates(
             .handle_fingerprint_change(context, message_time)
             .await?;
 
-        gossiped_addr.insert(header.addr.to_lowercase());
+        gossiped_keys.insert(header.addr.to_lowercase(), header.public_key);
     }
 
-    Ok(gossiped_addr)
+    Ok(gossiped_keys)
 }
 
 /// Message Disposition Notification (RFC 8098)
@@ -1820,6 +1850,8 @@ pub(crate) fn parse_message_id(ids: &str) -> Result<String> {
     }
 }
 
+/// Returns true if the header overwrites outer header
+/// when it comes from protected headers.
 fn is_known(key: &str) -> bool {
     matches!(
         key,
@@ -1835,6 +1867,7 @@ fn is_known(key: &str) -> bool {
             | "in-reply-to"
             | "references"
             | "subject"
+            | "secure-join"
     )
 }
 
@@ -2224,11 +2257,13 @@ mod tests {
 
     use super::*;
     use crate::{
+        chat,
         chatlist::Chatlist,
         constants::{Blocked, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS},
         message::{Message, MessageState, MessengerMessage},
         receive_imf::receive_imf,
-        test_utils::TestContext,
+        test_utils::{TestContext, TestContextManager},
+        tools::time,
     };
 
     impl AvatarAction {
@@ -2737,6 +2772,7 @@ Chat-Version: 1.0\n\
 Message-ID: <bar@example.org>\n\
 To: Alice <alice@example.org>\n\
 From: Bob <bob@example.org>\n\
+Auto-Submitted: auto-replied\n\
 Content-Type: multipart/report; report-type=disposition-notification;\n\t\
 boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
 \n\
@@ -2772,6 +2808,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
 
         assert_eq!(message.parts.len(), 1);
         assert_eq!(message.mdn_reports.len(), 1);
+        assert_eq!(message.is_bot, None);
     }
 
     /// Test parsing multiple MDNs combined in a single message.
@@ -3544,6 +3581,29 @@ On 2020-10-25, Bob wrote:
         );
     }
 
+    /// Tests that X-Microsoft-Original-Message-ID does not overwrite encrypted Message-ID.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_x_microsoft_original_message_id_precedence() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        let bob_chat_id = tcm.send_recv_accept(&alice, &bob, "hi").await.chat_id;
+        chat::send_text_msg(&bob, bob_chat_id, "hi!".to_string()).await?;
+        let mut sent_msg = bob.pop_sent_msg().await;
+
+        // Insert X-Microsoft-Original-Message-ID.
+        // It should be ignored because there is a Message-ID in the encrypted part.
+        sent_msg.payload = sent_msg.payload.replace(
+            "Message-ID:",
+            "X-Microsoft-Original-Message-ID: <fake-message-id@example.net>\r\nMessage-ID:",
+        );
+
+        let msg = alice.recv_msg(&sent_msg).await;
+        assert!(!msg.rfc724_mid.contains("fake-message-id"));
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_long_in_reply_to() -> Result<()> {
         let t = TestContext::new_alice().await;
@@ -3823,4 +3883,56 @@ Content-Disposition: reaction\n\
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_tlsrpt() -> Result<()> {
+        let context = TestContext::new_alice().await;
+        let raw = include_bytes!("../test-data/message/tlsrpt.eml");
+
+        let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
+            .await
+            .unwrap();
+        assert_eq!(msg.parts.len(), 1);
+
+        assert_eq!(msg.parts[0].typ, Viewtype::File);
+        assert_eq!(msg.parts[0].msg, "Report Domain: nine.testrun.org Submitter: google.com Report-ID: <2024.01.20T00.00.00Z+nine.testrun.org@google.com> – This is an aggregate TLS report from google.com");
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_time_in_future() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let beginning_time = time();
+
+        // Receive a message with a date far in the future (year 3004)
+        // I'm just going to assume that no one uses this code after the year 3000
+        let mime_message = MimeMessage::from_bytes(
+            &alice,
+            b"To: alice@example.org\n\
+              From: bob@example.net\n\
+              Date: Today, 29 February 3004 00:00:10 -800\n\
+              Message-ID: 56789@example.net\n\
+              Subject: Meeting\n\
+              Mime-Version: 1.0 (1.0)\n\
+              Content-Type: text/plain; charset=utf-8\n\
+              \n\
+              Hi",
+            None,
+        )
+        .await?;
+
+        // We do allow the time to be in the future a bit (because of unsynchronized clocks),
+        // but only 60 seconds:
+        assert!(mime_message.decryption_info.message_time <= time() + 60);
+        assert!(mime_message.decryption_info.message_time >= beginning_time + 60);
+        assert_eq!(
+            mime_message.decryption_info.message_time,
+            mime_message.timestamp_sent
+        );
+        assert!(mime_message.timestamp_rcvd <= time());
+
+        Ok(())
+    }
 }

src/param.rs

@@ -65,7 +65,7 @@ pub enum Param {
     /// For Messages: the message is a reaction.
     Reaction = b'x',
 
-    /// For Messages: a message with Auto-Submitted header ("bot").
+    /// For Messages: a message with "Auto-Submitted: auto-generated" header ("bot").
     Bot = b'b',
 
     /// For Messages: unset or 0=not forwarded,
@@ -87,7 +87,7 @@ pub enum Param {
     /// `Secure-Join-Fingerprint` header for `{vc,vg}-request-with-auth` messages.
     Arg3 = b'G',
 
-    /// For Messages
+    /// Deprecated `Secure-Join-Group` header for messages.
     Arg4 = b'H',
 
     /// For Messages

src/peer_channels.rs

@@ -1,413 +0,0 @@
-//! Peer channels for webxdc updates using iroh.
-
-use anyhow::{anyhow, Context as _, Result};
-use image::EncodableLayout;
-use iroh_base::base32;
-use iroh_gossip::net::{Gossip, GOSSIP_ALPN};
-use iroh_gossip::proto::{Event as IrohEvent, TopicId};
-use iroh_net::magic_endpoint::accept_conn;
-use iroh_net::NodeId;
-use iroh_net::{derp::DerpMode, key::SecretKey, MagicEndpoint};
-
-use crate::config::Config;
-use crate::contact::ContactId;
-use crate::context::Context;
-use crate::message::{Message, MsgId};
-use crate::tools::time;
-use crate::webxdc::StatusUpdateItem;
-
-impl Context {
-    /// Create magic endpoint and gossip for the context.
-    pub async fn create_gossip(&self) -> Result<()> {
-        let secret_key: SecretKey = self.get_or_generate_iroh_keypair().await?;
-        println!("> our secret key: {}", base32::fmt(secret_key.to_bytes()));
-
-        if self.endpoint.lock().await.is_some() {
-            warn!(
-                self,
-                "Tried to create endpoint even though there is already one."
-            );
-            return Ok(());
-        }
-
-        // build magic endpoint
-        let endpoint = MagicEndpoint::builder()
-            .secret_key(secret_key)
-            .alpns(vec![GOSSIP_ALPN.to_vec()])
-            .derp_mode(DerpMode::Default)
-            .bind(0)
-            .await?;
-
-        // create gossip
-        let my_addr = endpoint.my_addr().await?;
-        let gossip = Gossip::from_endpoint(endpoint.clone(), Default::default(), &my_addr.info);
-
-        // spawn endpoint loop that forwards incoming connections to the gossiper
-        let context = self.clone();
-        tokio::spawn(endpoint_loop(context, endpoint.clone(), gossip.clone()));
-
-        *self.gossip.lock().await = Some(gossip);
-        *self.endpoint.lock().await = Some(endpoint);
-        Ok(())
-    }
-
-    /// Join a topic and create the subscriber loop for it.
-    pub async fn join_and_subscribe_topic(&self, topic: TopicId, msg_id: MsgId) -> Result<()> {
-        info!(&self, "Joining topic {}.", topic.to_string());
-
-        let Some(ref gossip) = *self.gossip.lock().await else {
-            warn!(
-                self,
-                "Not joining topic {topic} because there is no gossip."
-            );
-            return Ok(());
-        };
-
-        // restore old peers from db, if any
-        let peers = self.get_peers_for_topic(topic).await?;
-        if peers.len() == 0 {
-            // TODO: When there's no peers we will never be able to join the gossip?
-            warn!(self, "joining gossip with zero peers");
-        } else {
-            info!(self, "joining gossip with peers: {peers:?}");
-            info!(
-                self,
-                "{:?}",
-                self.endpoint
-                    .lock()
-                    .await
-                    .as_ref()
-                    .unwrap()
-                    .my_addr()
-                    .await?
-            );
-        }
-
-        // TODO: add timeout as the returned future might be pending forever
-        let connect_future = gossip.join(topic, peers).await?;
-
-        tokio::spawn(connect_future);
-        tokio::spawn(subscribe_loop(self.clone(), gossip.clone(), topic, msg_id));
-
-        Ok(())
-    }
-
-    /// Get list of [NodeId]s for one topic.
-    /// This is used to rejoin a gossip group when reopening the xdc.
-    /// Only [NodeId] is needed because the magic endpoint caches region and derp server for [NodeId]s.
-    pub async fn get_peers_for_topic(&self, topic: TopicId) -> Result<Vec<NodeId>> {
-        self.sql
-            .query_map(
-                "SELECT public_key FROM iroh_gossip_peers WHERE topic = ?",
-                (topic.as_bytes(),),
-                |row| {
-                    let data = row.get::<_, Vec<u8>>(0)?;
-                    Ok(data)
-                },
-                |g| {
-                    g.map(|data| {
-                        Ok::<NodeId, anyhow::Error>(NodeId::from_bytes(
-                            &data?
-                                .try_into()
-                                .map_err(|_| anyhow!("Can't convert sql data to [u8; 32]"))?,
-                        )?)
-                    })
-                    .collect::<std::result::Result<Vec<_>, _>>()
-                    .map_err(Into::into)
-                },
-            )
-            .await
-    }
-
-    /// Cache a peers [NodeId] for one topic.
-    pub async fn add_peer_for_topic(
-        &self,
-        msg_id: MsgId,
-        topic: TopicId,
-        peer: NodeId,
-    ) -> Result<()> {
-        self.sql
-            .execute(
-                "INSERT INTO iroh_gossip_peers (msg_id, public_key, topic) VALUES (?, ?, ?)",
-                (msg_id, peer.as_bytes(), topic.as_bytes()),
-            )
-            .await?;
-        Ok(())
-    }
-
-    /// Remove one cached peer from a topic.
-    pub async fn delete_peer_for_topic(&self, topic: TopicId, peer: NodeId) -> Result<()> {
-        self.sql
-            .execute(
-                "DELETE FROM iroh_gossip_peers WHERE public_key = ? topic = ?",
-                (peer.as_bytes(), topic.as_bytes()),
-            )
-            .await?;
-        Ok(())
-    }
-
-    /// Get the iroh gossip secret key from the database or generate a new one and persist it.
-    pub async fn get_or_generate_iroh_keypair(&self) -> Result<SecretKey> {
-        match self.get_config_parsed(Config::IrohSecretKey).await? {
-            Some(key) => Ok(key),
-            None => {
-                let key = SecretKey::generate();
-                self.set_config(Config::IrohSecretKey, Some(&key.to_string()))
-                    .await?;
-                Ok(key)
-            }
-        }
-    }
-}
-
-async fn endpoint_loop(context: Context, endpoint: MagicEndpoint, gossip: Gossip) {
-    while let Some(conn) = endpoint.accept().await {
-        info!(context, "accepting connection with {:?}", conn);
-        let gossip = gossip.clone();
-        let context = context.clone();
-        tokio::spawn(async move {
-            if let Err(err) = handle_connection(&context, conn, gossip).await {
-                warn!(context, "iroh connection error: {err}");
-            }
-        });
-    }
-}
-
-async fn handle_connection(
-    context: &Context,
-    conn: quinn::Connecting,
-    gossip: Gossip,
-) -> anyhow::Result<()> {
-    let (peer_id, alpn, conn) = accept_conn(conn).await?;
-    match alpn.as_bytes() {
-        GOSSIP_ALPN => gossip
-            .handle_connection(conn)
-            .await
-            .context(format!("Connection to {peer_id} with ALPN {alpn} failed"))?,
-        _ => info!(
-            context,
-            "Ignoring connection from {peer_id}: unsupported ALPN protocol"
-        ),
-    }
-    Ok(())
-}
-
-async fn subscribe_loop(
-    context: Context,
-    gossip: Gossip,
-    topic: TopicId,
-    msg_id: MsgId,
-) -> Result<()> {
-    let mut stream = gossip.subscribe(topic).await?;
-    loop {
-        let event = stream.recv().await?;
-        match event {
-            IrohEvent::NeighborUp(node) => {
-                info!(context, "NeighborUp: {:?}", node);
-                context.add_peer_for_topic(msg_id, topic, node).await?;
-            }
-            IrohEvent::NeighborDown(node) => {
-                info!(context, "NeighborDown: {:?}", node);
-                context.delete_peer_for_topic(topic, node).await?;
-            }
-            IrohEvent::Received(event) => {
-                info!(context, "Received: {:?}", event);
-                let payload = String::from_utf8_lossy(event.content.as_bytes());
-                let mut instance = Message::load_from_db(&context, msg_id).await?;
-                let update: StatusUpdateItem = serde_json::from_str(&payload)?;
-                context
-                    .create_status_update_record(
-                        &mut instance,
-                        update,
-                        time(),
-                        false,
-                        ContactId::SELF,
-                    )
-                    .await?;
-            }
-        };
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::{os::unix::thread, str::FromStr, time::Duration};
-
-    use tokio::time::timeout;
-
-    use crate::{
-        chat::send_msg,
-        message::Viewtype,
-        test_utils::TestContextManager,
-        webxdc::{join_gossip_topic, StatusUpdateSerial},
-        EventType,
-    };
-
-    use super::*;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_can_connect() {
-        let mut tcm = TestContextManager::new();
-        let alice = &mut tcm.alice().await;
-        let bob = &mut tcm.bob().await;
-        alice.ctx.start_io().await;
-        bob.ctx.start_io().await;
-
-        // Alice sends webxdc to bob
-        let alice_chat = alice.create_chat(bob).await;
-        let mut instance = Message::new(Viewtype::File);
-        instance
-            .set_file_from_bytes(
-                alice,
-                "minimal.xdc",
-                include_bytes!("../test-data/webxdc/minimal.xdc"),
-                None,
-            )
-            .await
-            .unwrap();
-
-        send_msg(alice, alice_chat.id, &mut instance).await.unwrap();
-
-        let alice_instance = alice.get_last_msg().await;
-        assert_eq!(alice_instance.get_viewtype(), Viewtype::Webxdc);
-
-        let webxdc = alice.pop_sent_msg().await;
-        let bob_webdxc = bob.recv_msg(&webxdc).await;
-        bob_webdxc.chat_id.accept(bob).await.unwrap();
-
-        assert_eq!(bob_webdxc.get_viewtype(), Viewtype::Webxdc);
-
-        // Alice sends webxdc update with gossip.
-        // This produces an SMTP message that contains the topic and a header with alices' node id
-        alice
-            .send_webxdc_status_update_struct(
-                alice_instance.id,
-                StatusUpdateItem {
-                    payload: "test".to_string().into(),
-                    gossip_topic: Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
-                    ..Default::default()
-                },
-                "",
-            )
-            .await
-            .unwrap();
-
-        alice.flush_status_updates().await.unwrap();
-        bob.recv_msg(&alice.pop_sent_msg().await).await;
-
-        let status = bob
-            .get_webxdc_status_updates(bob_webdxc.id, StatusUpdateSerial::new(0))
-            .await
-            .unwrap();
-        let status_update_items: Vec<StatusUpdateItem> = serde_json::from_str(&status).unwrap();
-        let topic = status_update_items[0].gossip_topic.as_ref().unwrap();
-        assert_eq!(topic, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-
-        let topic_id = TopicId::from_str(&iroh_base::base32::fmt(topic)).unwrap();
-        let topics = bob.get_peers_for_topic(topic_id).await.unwrap();
-        assert_eq!(
-            topics,
-            vec![alice.endpoint.lock().await.as_ref().unwrap().node_id()]
-        );
-
-        let mut stream = alice
-            .ctx
-            .gossip
-            .lock()
-            .await
-            .as_ref()
-            .unwrap()
-            .subscribe(topic_id)
-            .await
-            .unwrap();
-
-        // Bob joins topic
-        join_gossip_topic(bob, bob_webdxc.id, topic).await.unwrap();
-
-        let event = timeout(Duration::from_secs(5), stream.recv())
-            .await
-            .unwrap()
-            .unwrap();
-        match event {
-            IrohEvent::NeighborUp(node) => {
-                assert_eq!(node, bob.endpoint.lock().await.as_ref().unwrap().node_id());
-            }
-            _ => panic!("Expected NeighborUp event"),
-        }
-
-        // Bob sends webxdc update with gossip.
-        bob.send_webxdc_status_update_struct(
-            bob_webdxc.id,
-            StatusUpdateItem {
-                payload: "bob -> alice".to_string().into(),
-                gossip_topic: Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
-                ..Default::default()
-            },
-            "",
-        )
-        .await
-        .unwrap();
-
-        alice.evtracker.try_recv().unwrap();
-        while let Ok(event) = alice.evtracker.try_recv() {
-            if let EventType::WebxdcStatusUpdate {
-                msg_id,
-                status_update_serial,
-            } = event.typ
-            {
-                let status_update = alice
-                    .get_status_update(msg_id, status_update_serial)
-                    .await
-                    .unwrap();
-                let status_update_item: StatusUpdateItem =
-                    serde_json::from_str(&status_update).unwrap();
-                println!("{:?}", status_update_item.payload.to_string());
-                if status_update_item
-                    .payload
-                    .to_string()
-                    .contains("bob -> alice")
-                {
-                    break;
-                }
-            }
-        }
-
-        // Alice sends webxdc update with gossip.
-        alice
-            .send_webxdc_status_update_struct(
-                bob_webdxc.id,
-                StatusUpdateItem {
-                    payload: "alice -> bob".to_string().into(),
-                    gossip_topic: Some("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string()),
-                    ..Default::default()
-                },
-                "",
-            )
-            .await
-            .unwrap();
-
-        while let Ok(event) = bob.evtracker.try_recv() {
-            if let EventType::WebxdcStatusUpdate {
-                msg_id,
-                status_update_serial,
-            } = event.typ
-            {
-                let status_update = alice
-                    .get_status_update(msg_id, status_update_serial)
-                    .await
-                    .unwrap();
-
-                let status_update_item: StatusUpdateItem =
-                    serde_json::from_str(&status_update).unwrap();
-
-                if status_update_item
-                    .payload
-                    .to_string()
-                    .contains("alice -> bob")
-                {
-                    break;
-                }
-            }
-        }
-    }
-}

src/peerstate.rs

@@ -16,6 +16,7 @@ use crate::message::Message;
 use crate::mimeparser::SystemMessage;
 use crate::sql::Sql;
 use crate::stock_str;
+use crate::tools;
 
 /// Type of the public key stored inside the peerstate.
 #[derive(Debug)]
@@ -97,13 +98,28 @@ pub struct Peerstate {
 impl Peerstate {
     /// Creates a peerstate from the `Autocrypt` header.
     pub fn from_header(header: &Aheader, message_time: i64) -> Self {
+        Self::from_public_key(
+            &header.addr,
+            message_time,
+            header.prefer_encrypt,
+            &header.public_key,
+        )
+    }
+
+    /// Creates a peerstate from the given public key.
+    pub fn from_public_key(
+        addr: &str,
+        last_seen: i64,
+        prefer_encrypt: EncryptPreference,
+        public_key: &SignedPublicKey,
+    ) -> Self {
         Peerstate {
-            addr: header.addr.clone(),
-            last_seen: message_time,
-            last_seen_autocrypt: message_time,
-            prefer_encrypt: header.prefer_encrypt,
-            public_key: Some(header.public_key.clone()),
-            public_key_fingerprint: Some(header.public_key.fingerprint()),
+            addr: addr.to_string(),
+            last_seen,
+            last_seen_autocrypt: last_seen,
+            prefer_encrypt,
+            public_key: Some(public_key.clone()),
+            public_key_fingerprint: Some(public_key.fingerprint()),
             gossip_key: None,
             gossip_key_fingerprint: None,
             gossip_timestamp: 0,
@@ -150,6 +166,9 @@ impl Peerstate {
 
     /// Loads peerstate corresponding to the given address from the database.
     pub async fn from_addr(context: &Context, addr: &str) -> Result<Option<Peerstate>> {
+        if context.is_self_addr(addr).await? {
+            return Ok(Some(Peerstate::get_self_stub(addr)));
+        }
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint, \
@@ -167,6 +186,7 @@ impl Peerstate {
         context: &Context,
         fingerprint: &Fingerprint,
     ) -> Result<Option<Peerstate>> {
+        // NOTE: If it's our key fingerprint, this returns None currently.
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint, \
@@ -191,6 +211,9 @@ impl Peerstate {
         fingerprint: &Fingerprint,
         addr: &str,
     ) -> Result<Option<Peerstate>> {
+        if context.is_self_addr(addr).await? {
+            return Ok(Some(Peerstate::get_self_stub(addr)));
+        }
         let query = "SELECT addr, last_seen, last_seen_autocrypt, prefer_encrypted, public_key, \
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint, \
@@ -206,6 +229,34 @@ impl Peerstate {
         Self::from_stmt(context, query, (&fp, &addr, &fp)).await
     }
 
+    /// Returns peerstate stub for self `addr`.
+    ///
+    /// Needed for [`crate::decrypt::keyring_from_peerstate()`] which returns a keyring of all our
+    /// pubkeys for such a stub so that we can check if a message is signed by us.
+    fn get_self_stub(addr: &str) -> Self {
+        let now = tools::time();
+        // We can have multiple pubkeys, just make the corresponding fields None.
+        Self {
+            addr: addr.to_string(),
+            last_seen: now,
+            last_seen_autocrypt: now,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: None,
+            public_key_fingerprint: None,
+            gossip_key: None,
+            gossip_key_fingerprint: None,
+            gossip_timestamp: 0,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            verifier: None,
+            secondary_verified_key: None,
+            secondary_verified_key_fingerprint: None,
+            secondary_verifier: None,
+            backward_verified_key_id: None,
+            fingerprint_changed: false,
+        }
+    }
+
     async fn from_stmt(
         context: &Context,
         query: &str,
@@ -322,7 +373,7 @@ impl Peerstate {
             return;
         }
 
-        if message_time > self.last_seen {
+        if message_time >= self.last_seen {
             self.last_seen = message_time;
             self.last_seen_autocrypt = message_time;
             if (header.prefer_encrypt == EncryptPreference::Mutual
@@ -345,7 +396,7 @@ impl Peerstate {
             return;
         }
 
-        if message_time > self.gossip_timestamp {
+        if message_time >= self.gossip_timestamp {
             self.gossip_timestamp = message_time;
             if self.gossip_key.as_ref() != Some(&gossip_header.public_key) {
                 self.gossip_key = Some(gossip_header.public_key.clone());
@@ -457,59 +508,44 @@ impl Peerstate {
         Ok(backward_verified)
     }
 
-    /// Set this peerstate to verified
-    /// Make sure to call `self.save_to_db` to save these changes
+    /// Set this peerstate to verified;
+    /// make sure to call `self.save_to_db` to save these changes.
+    ///
     /// Params:
-    /// verifier:
+    ///
+    /// * key: The new verified key.
+    /// * fingerprint: Only set to verified if the key's fingerprint matches this.
+    /// * verifier:
     ///   The address which introduces the given contact.
     ///   If we are verifying the contact, use that contacts address.
     pub fn set_verified(
         &mut self,
-        which_key: PeerstateKeyType,
+        key: SignedPublicKey,
         fingerprint: Fingerprint,
         verifier: String,
     ) -> Result<()> {
-        match which_key {
-            PeerstateKeyType::PublicKey => {
-                if self.public_key_fingerprint.is_some()
-                    && self.public_key_fingerprint.as_ref().unwrap() == &fingerprint
-                {
-                    self.verified_key = self.public_key.clone();
-                    self.verified_key_fingerprint = Some(fingerprint);
-                    self.verifier = Some(verifier);
-                    Ok(())
-                } else {
-                    Err(Error::msg(format!(
-                        "{fingerprint} is not peer's public key fingerprint",
-                    )))
-                }
-            }
-            PeerstateKeyType::GossipKey => {
-                if self.gossip_key_fingerprint.is_some()
-                    && self.gossip_key_fingerprint.as_ref().unwrap() == &fingerprint
-                {
-                    self.verified_key = self.gossip_key.clone();
-                    self.verified_key_fingerprint = Some(fingerprint);
-                    self.verifier = Some(verifier);
-                    Ok(())
-                } else {
-                    Err(Error::msg(format!(
-                        "{fingerprint} is not peer's gossip key fingerprint",
-                    )))
-                }
-            }
+        if key.fingerprint() == fingerprint {
+            self.verified_key = Some(key);
+            self.verified_key_fingerprint = Some(fingerprint);
+            self.verifier = Some(verifier);
+            Ok(())
+        } else {
+            Err(Error::msg(format!(
+                "{fingerprint} is not peer's key fingerprint",
+            )))
         }
     }
 
-    /// Sets current gossiped key as the secondary verified key.
+    /// Sets the gossiped key as the secondary verified key.
     ///
     /// If gossiped key is the same as the current verified key,
     /// do nothing to avoid overwriting secondary verified key
     /// which may be different.
-    pub fn set_secondary_verified_key_from_gossip(&mut self, verifier: String) {
-        if self.gossip_key_fingerprint != self.verified_key_fingerprint {
-            self.secondary_verified_key = self.gossip_key.clone();
-            self.secondary_verified_key_fingerprint = self.gossip_key_fingerprint.clone();
+    pub fn set_secondary_verified_key(&mut self, gossip_key: SignedPublicKey, verifier: String) {
+        let fingerprint = gossip_key.fingerprint();
+        if self.verified_key_fingerprint.as_ref() != Some(&fingerprint) {
+            self.secondary_verified_key = Some(gossip_key);
+            self.secondary_verified_key_fingerprint = Some(fingerprint);
             self.secondary_verifier = Some(verifier);
         }
     }
@@ -995,7 +1031,7 @@ mod tests {
         assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Reset);
 
         // Same header will be applied in the future.
-        peerstate.apply_header(&header, 400);
+        peerstate.apply_header(&header, 300);
         assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual);
     }
 }

src/pgp.rs

@@ -4,7 +4,7 @@ use std::collections::{BTreeMap, HashSet};
 use std::io;
 use std::io::Cursor;
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, Context as _, Result};
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
@@ -138,8 +138,11 @@ pub struct KeyPair {
 }
 
 /// Create a new key pair.
+///
+/// Both secret and public key consist of signing primary key and encryption subkey
+/// as [described in the Autocrypt standard](https://autocrypt.org/level1.html#openpgp-based-key-data).
 pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Result<KeyPair> {
-    let (secret_key_type, public_key_type) = match keygen_type {
+    let (signing_key_type, encryption_key_type) = match keygen_type {
         KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
         KeyGenType::Rsa4096 => (PgpKeyType::Rsa(4096), PgpKeyType::Rsa(4096)),
         KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
@@ -147,8 +150,8 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
 
     let user_id = format!("<{addr}>");
     let key_params = SecretKeyParamsBuilder::default()
-        .key_type(secret_key_type)
-        .can_create_certificates(true)
+        .key_type(signing_key_type)
+        .can_certify(true)
         .can_sign(true)
         .primary_user_id(user_id)
         .passphrase(None)
@@ -170,41 +173,36 @@ pub(crate) fn create_keypair(addr: EmailAddress, keygen_type: KeyGenType) -> Res
         ])
         .subkey(
             SubkeyParamsBuilder::default()
-                .key_type(public_key_type)
+                .key_type(encryption_key_type)
                 .can_encrypt(true)
                 .passphrase(None)
                 .build()
-                .map_err(|e| format_err!("{}", e))
                 .context("failed to build subkey parameters")?,
         )
         .build()
-        .map_err(|e| format_err!("{}", e))
-        .context("invalid key params")?;
-    let key = key_params.generate().context("invalid params")?;
-    let private_key = key
+        .context("failed to build key parameters")?;
+
+    let secret_key = key_params
+        .generate()
+        .context("failed to generate the key")?
         .sign(|| "".into())
-        .map_err(|e| format_err!("{}", e))
         .context("failed to sign secret key")?;
-
-    let public_key = private_key.public_key();
-    let public_key = public_key
-        .sign(&private_key, || "".into())
-        .map_err(|e| format_err!("{}", e))
-        .context("failed to sign public key")?;
-
-    private_key
+    secret_key
         .verify()
-        .map_err(|e| format_err!("{}", e))
-        .context("invalid private key generated")?;
+        .context("invalid secret key generated")?;
+
+    let public_key = secret_key
+        .public_key()
+        .sign(&secret_key, || "".into())
+        .context("failed to sign public key")?;
     public_key
         .verify()
-        .map_err(|e| format_err!("{}", e))
         .context("invalid public key generated")?;
 
     Ok(KeyPair {
         addr,
         public: public_key,
-        secret: private_key,
+        secret: secret_key,
     })
 }
 
@@ -238,6 +236,7 @@ pub async fn pk_encrypt(
     plain: &[u8],
     public_keys_for_encryption: Vec<SignedPublicKey>,
     private_key_for_signing: Option<SignedSecretKey>,
+    compress: bool,
 ) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
 
@@ -251,20 +250,19 @@ pub async fn pk_encrypt(
 
             let mut rng = thread_rng();
 
-            // TODO: measure time
             let encrypted_msg = if let Some(ref skey) = private_key_for_signing {
-                lit_msg
-                    .sign(skey, || "".into(), HASH_ALGORITHM)
-                    .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-                    .and_then(|msg| {
-                        msg.encrypt_to_keys(&mut rng, SYMMETRIC_KEY_ALGORITHM, &pkeys_refs)
-                    })
+                let signed_msg = lit_msg.sign(skey, || "".into(), HASH_ALGORITHM)?;
+                let compressed_msg = if compress {
+                    signed_msg.compress(CompressionAlgorithm::ZLIB)?
+                } else {
+                    signed_msg
+                };
+                compressed_msg.encrypt_to_keys(&mut rng, SYMMETRIC_KEY_ALGORITHM, &pkeys_refs)?
             } else {
-                lit_msg.encrypt_to_keys(&mut rng, SYMMETRIC_KEY_ALGORITHM, &pkeys_refs)
+                lit_msg.encrypt_to_keys(&mut rng, SYMMETRIC_KEY_ALGORITHM, &pkeys_refs)?
             };
 
-            let msg = encrypted_msg?;
-            let encoded_msg = msg.to_armored_string(None)?;
+            let encoded_msg = encrypted_msg.to_armored_string(None)?;
 
             Ok(encoded_msg)
         })
@@ -486,10 +484,16 @@ mod tests {
         CTEXT_SIGNED
             .get_or_init(|| async {
                 let keyring = vec![KEYS.alice_public.clone(), KEYS.bob_public.clone()];
+                let compress = true;
 
-                pk_encrypt(CLEARTEXT, keyring, Some(KEYS.alice_secret.clone()))
-                    .await
-                    .unwrap()
+                pk_encrypt(
+                    CLEARTEXT,
+                    keyring,
+                    Some(KEYS.alice_secret.clone()),
+                    compress,
+                )
+                .await
+                .unwrap()
             })
             .await
     }
@@ -499,7 +503,11 @@ mod tests {
         CTEXT_UNSIGNED
             .get_or_init(|| async {
                 let keyring = vec![KEYS.alice_public.clone(), KEYS.bob_public.clone()];
-                pk_encrypt(CLEARTEXT, keyring, None).await.unwrap()
+                let compress = true;
+
+                pk_encrypt(CLEARTEXT, keyring, None, compress)
+                    .await
+                    .unwrap()
             })
             .await
     }

src/provider.rs

@@ -215,8 +215,18 @@ pub async fn get_provider_info(
 
 /// Finds a provider in offline database based on domain.
 pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
-    if let Some(provider) = PROVIDER_DATA.get(domain.to_lowercase().as_str()) {
-        return Some(*provider);
+    let domain = domain.to_lowercase();
+    for (pattern, provider) in PROVIDER_DATA {
+        if let Some(suffix) = pattern.strip_prefix('*') {
+            // Wildcard domain pattern.
+            //
+            // For example, `suffix` is ".hermes.radio" for "*.hermes.radio" pattern.
+            if domain.ends_with(suffix) {
+                return Some(provider);
+            }
+        } else if pattern == domain {
+            return Some(provider);
+        }
     }
 
     None
@@ -226,33 +236,42 @@ pub fn get_provider_by_domain(domain: &str) -> Option<&'static Provider> {
 ///
 /// For security reasons, only Gmail can be configured this way.
 pub async fn get_provider_by_mx(context: &Context, domain: &str) -> Option<&'static Provider> {
-    if let Ok(resolver) = get_resolver() {
-        let mut fqdn: String = domain.to_string();
-        if !fqdn.ends_with('.') {
-            fqdn.push('.');
+    let Ok(resolver) = get_resolver() else {
+        warn!(context, "Cannot get a resolver to check MX records.");
+        return None;
+    };
+
+    let mut fqdn: String = domain.to_string();
+    if !fqdn.ends_with('.') {
+        fqdn.push('.');
+    }
+
+    let Ok(mx_domains) = resolver.mx_lookup(fqdn).await else {
+        warn!(context, "Cannot resolve MX records for {domain:?}.");
+        return None;
+    };
+
+    for (provider_domain_pattern, provider) in PROVIDER_DATA {
+        if provider.id != "gmail" {
+            // MX lookup is limited to Gmail for security reasons
+            continue;
         }
 
-        if let Ok(mx_domains) = resolver.mx_lookup(fqdn).await {
-            for (provider_domain, provider) in &*PROVIDER_DATA {
-                if provider.id != "gmail" {
-                    // MX lookup is limited to Gmail for security reasons
-                    continue;
-                }
+        if provider_domain_pattern.starts_with('*') {
+            // Skip wildcard patterns.
+            continue;
+        }
 
-                let provider_fqdn = provider_domain.to_string() + ".";
-                let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
+        let provider_fqdn = provider_domain_pattern.to_string() + ".";
+        let provider_fqdn_dot = ".".to_string() + &provider_fqdn;
 
-                for mx_domain in mx_domains.iter() {
-                    let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
+        for mx_domain in mx_domains.iter() {
+            let mx_domain = mx_domain.exchange().to_lowercase().to_utf8();
 
-                    if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
-                        return Some(provider);
-                    }
-                }
+            if mx_domain == provider_fqdn || mx_domain.ends_with(&provider_fqdn_dot) {
+                return Some(provider);
             }
         }
-    } else {
-        warn!(context, "cannot get a resolver to check MX records.");
     }
 
     None

src/provider/data.rs

@@ -593,7 +593,7 @@ static P_GMX_NET: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-// hermes.radio.md: ac.hermes.radio, ac1.hermes.radio, ac2.hermes.radio, ac3.hermes.radio, ac4.hermes.radio, ac5.hermes.radio, ac6.hermes.radio, ac7.hermes.radio, ac8.hermes.radio, ac9.hermes.radio, ac10.hermes.radio, ac11.hermes.radio, ac12.hermes.radio, ac13.hermes.radio, ac14.hermes.radio, ac15.hermes.radio, ka.hermes.radio, ka1.hermes.radio, ka2.hermes.radio, ka3.hermes.radio, ka4.hermes.radio, ka5.hermes.radio, ka6.hermes.radio, ka7.hermes.radio, ka8.hermes.radio, ka9.hermes.radio, ka10.hermes.radio, ka11.hermes.radio, ka12.hermes.radio, ka13.hermes.radio, ka14.hermes.radio, ka15.hermes.radio, ec.hermes.radio, ec1.hermes.radio, ec2.hermes.radio, ec3.hermes.radio, ec4.hermes.radio, ec5.hermes.radio, ec6.hermes.radio, ec7.hermes.radio, ec8.hermes.radio, ec9.hermes.radio, ec10.hermes.radio, ec11.hermes.radio, ec12.hermes.radio, ec13.hermes.radio, ec14.hermes.radio, ec15.hermes.radio, hermes.radio
+// hermes.radio.md: *.hermes.radio, *.aco-connexion.org
 static P_HERMES_RADIO: Provider = Provider {
     id: "hermes.radio",
     status: Status::Ok,
@@ -1608,375 +1608,326 @@ static P_ZOHO: Provider = Provider {
     oauth2_authorizer: None,
 };
 
-pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
-    HashMap::from([
-        ("163.com", &P_163),
-        ("aktivix.org", &P_AKTIVIX_ORG),
-        ("aol.com", &P_AOL),
-        ("arcor.de", &P_ARCOR_DE),
-        ("autistici.org", &P_AUTISTICI_ORG),
-        ("delta.blinzeln.de", &P_BLINDZELN_ORG),
-        ("delta.blindzeln.org", &P_BLINDZELN_ORG),
-        ("bluewin.ch", &P_BLUEWIN_CH),
-        ("buzon.uy", &P_BUZON_UY),
-        ("c1.testrun.org", &P_C1_TESTRUN_ORG),
-        ("c2.testrun.org", &P_C2_TESTRUN_ORG),
-        ("c3.testrun.org", &P_C3_TESTRUN_ORG),
-        ("chello.at", &P_CHELLO_AT),
-        ("xfinity.com", &P_COMCAST),
-        ("comcast.net", &P_COMCAST),
-        ("dismail.de", &P_DISMAIL_DE),
-        ("disroot.org", &P_DISROOT),
-        ("e.email", &P_E_EMAIL),
-        ("espiv.net", &P_ESPIV_NET),
-        ("example.com", &P_EXAMPLE_COM),
-        ("example.org", &P_EXAMPLE_COM),
-        ("example.net", &P_EXAMPLE_COM),
-        ("123mail.org", &P_FASTMAIL),
-        ("150mail.com", &P_FASTMAIL),
-        ("150ml.com", &P_FASTMAIL),
-        ("16mail.com", &P_FASTMAIL),
-        ("2-mail.com", &P_FASTMAIL),
-        ("4email.net", &P_FASTMAIL),
-        ("50mail.com", &P_FASTMAIL),
-        ("airpost.net", &P_FASTMAIL),
-        ("allmail.net", &P_FASTMAIL),
-        ("bestmail.us", &P_FASTMAIL),
-        ("cluemail.com", &P_FASTMAIL),
-        ("elitemail.org", &P_FASTMAIL),
-        ("emailcorner.net", &P_FASTMAIL),
-        ("emailengine.net", &P_FASTMAIL),
-        ("emailengine.org", &P_FASTMAIL),
-        ("emailgroups.net", &P_FASTMAIL),
-        ("emailplus.org", &P_FASTMAIL),
-        ("emailuser.net", &P_FASTMAIL),
-        ("eml.cc", &P_FASTMAIL),
-        ("f-m.fm", &P_FASTMAIL),
-        ("fast-email.com", &P_FASTMAIL),
-        ("fast-mail.org", &P_FASTMAIL),
-        ("fastem.com", &P_FASTMAIL),
-        ("fastemail.us", &P_FASTMAIL),
-        ("fastemailer.com", &P_FASTMAIL),
-        ("fastest.cc", &P_FASTMAIL),
-        ("fastimap.com", &P_FASTMAIL),
-        ("fastmail.cn", &P_FASTMAIL),
-        ("fastmail.co.uk", &P_FASTMAIL),
-        ("fastmail.com", &P_FASTMAIL),
-        ("fastmail.com.au", &P_FASTMAIL),
-        ("fastmail.de", &P_FASTMAIL),
-        ("fastmail.es", &P_FASTMAIL),
-        ("fastmail.fm", &P_FASTMAIL),
-        ("fastmail.fr", &P_FASTMAIL),
-        ("fastmail.im", &P_FASTMAIL),
-        ("fastmail.in", &P_FASTMAIL),
-        ("fastmail.jp", &P_FASTMAIL),
-        ("fastmail.mx", &P_FASTMAIL),
-        ("fastmail.net", &P_FASTMAIL),
-        ("fastmail.nl", &P_FASTMAIL),
-        ("fastmail.org", &P_FASTMAIL),
-        ("fastmail.se", &P_FASTMAIL),
-        ("fastmail.to", &P_FASTMAIL),
-        ("fastmail.tw", &P_FASTMAIL),
-        ("fastmail.uk", &P_FASTMAIL),
-        ("fastmail.us", &P_FASTMAIL),
-        ("fastmailbox.net", &P_FASTMAIL),
-        ("fastmessaging.com", &P_FASTMAIL),
-        ("fea.st", &P_FASTMAIL),
-        ("fmail.co.uk", &P_FASTMAIL),
-        ("fmailbox.com", &P_FASTMAIL),
-        ("fmgirl.com", &P_FASTMAIL),
-        ("fmguy.com", &P_FASTMAIL),
-        ("ftml.net", &P_FASTMAIL),
-        ("h-mail.us", &P_FASTMAIL),
-        ("hailmail.net", &P_FASTMAIL),
-        ("imap-mail.com", &P_FASTMAIL),
-        ("imap.cc", &P_FASTMAIL),
-        ("imapmail.org", &P_FASTMAIL),
-        ("inoutbox.com", &P_FASTMAIL),
-        ("internet-e-mail.com", &P_FASTMAIL),
-        ("internet-mail.org", &P_FASTMAIL),
-        ("internetemails.net", &P_FASTMAIL),
-        ("internetmailing.net", &P_FASTMAIL),
-        ("jetemail.net", &P_FASTMAIL),
-        ("justemail.net", &P_FASTMAIL),
-        ("letterboxes.org", &P_FASTMAIL),
-        ("mail-central.com", &P_FASTMAIL),
-        ("mail-page.com", &P_FASTMAIL),
-        ("mailandftp.com", &P_FASTMAIL),
-        ("mailas.com", &P_FASTMAIL),
-        ("mailbolt.com", &P_FASTMAIL),
-        ("mailc.net", &P_FASTMAIL),
-        ("mailcan.com", &P_FASTMAIL),
-        ("mailforce.net", &P_FASTMAIL),
-        ("mailftp.com", &P_FASTMAIL),
-        ("mailhaven.com", &P_FASTMAIL),
-        ("mailingaddress.org", &P_FASTMAIL),
-        ("mailite.com", &P_FASTMAIL),
-        ("mailmight.com", &P_FASTMAIL),
-        ("mailnew.com", &P_FASTMAIL),
-        ("mailsent.net", &P_FASTMAIL),
-        ("mailservice.ms", &P_FASTMAIL),
-        ("mailup.net", &P_FASTMAIL),
-        ("mailworks.org", &P_FASTMAIL),
-        ("ml1.net", &P_FASTMAIL),
-        ("mm.st", &P_FASTMAIL),
-        ("myfastmail.com", &P_FASTMAIL),
-        ("mymacmail.com", &P_FASTMAIL),
-        ("nospammail.net", &P_FASTMAIL),
-        ("ownmail.net", &P_FASTMAIL),
-        ("petml.com", &P_FASTMAIL),
-        ("postinbox.com", &P_FASTMAIL),
-        ("postpro.net", &P_FASTMAIL),
-        ("proinbox.com", &P_FASTMAIL),
-        ("promessage.com", &P_FASTMAIL),
-        ("realemail.net", &P_FASTMAIL),
-        ("reallyfast.biz", &P_FASTMAIL),
-        ("reallyfast.info", &P_FASTMAIL),
-        ("rushpost.com", &P_FASTMAIL),
-        ("sent.as", &P_FASTMAIL),
-        ("sent.at", &P_FASTMAIL),
-        ("sent.com", &P_FASTMAIL),
-        ("speedpost.net", &P_FASTMAIL),
-        ("speedymail.org", &P_FASTMAIL),
-        ("ssl-mail.com", &P_FASTMAIL),
-        ("swift-mail.com", &P_FASTMAIL),
-        ("the-fastest.net", &P_FASTMAIL),
-        ("the-quickest.com", &P_FASTMAIL),
-        ("theinternetemail.com", &P_FASTMAIL),
-        ("veryfast.biz", &P_FASTMAIL),
-        ("veryspeedy.net", &P_FASTMAIL),
-        ("warpmail.net", &P_FASTMAIL),
-        ("xsmail.com", &P_FASTMAIL),
-        ("yepmail.net", &P_FASTMAIL),
-        ("your-mail.com", &P_FASTMAIL),
-        ("firemail.at", &P_FIREMAIL_DE),
-        ("firemail.de", &P_FIREMAIL_DE),
-        ("five.chat", &P_FIVE_CHAT),
-        ("freenet.de", &P_FREENET_DE),
-        ("gmail.com", &P_GMAIL),
-        ("googlemail.com", &P_GMAIL),
-        ("google.com", &P_GMAIL),
-        ("gmx.net", &P_GMX_NET),
-        ("gmx.de", &P_GMX_NET),
-        ("gmx.at", &P_GMX_NET),
-        ("gmx.ch", &P_GMX_NET),
-        ("gmx.org", &P_GMX_NET),
-        ("gmx.eu", &P_GMX_NET),
-        ("gmx.info", &P_GMX_NET),
-        ("gmx.biz", &P_GMX_NET),
-        ("gmx.com", &P_GMX_NET),
-        ("ac.hermes.radio", &P_HERMES_RADIO),
-        ("ac1.hermes.radio", &P_HERMES_RADIO),
-        ("ac2.hermes.radio", &P_HERMES_RADIO),
-        ("ac3.hermes.radio", &P_HERMES_RADIO),
-        ("ac4.hermes.radio", &P_HERMES_RADIO),
-        ("ac5.hermes.radio", &P_HERMES_RADIO),
-        ("ac6.hermes.radio", &P_HERMES_RADIO),
-        ("ac7.hermes.radio", &P_HERMES_RADIO),
-        ("ac8.hermes.radio", &P_HERMES_RADIO),
-        ("ac9.hermes.radio", &P_HERMES_RADIO),
-        ("ac10.hermes.radio", &P_HERMES_RADIO),
-        ("ac11.hermes.radio", &P_HERMES_RADIO),
-        ("ac12.hermes.radio", &P_HERMES_RADIO),
-        ("ac13.hermes.radio", &P_HERMES_RADIO),
-        ("ac14.hermes.radio", &P_HERMES_RADIO),
-        ("ac15.hermes.radio", &P_HERMES_RADIO),
-        ("ka.hermes.radio", &P_HERMES_RADIO),
-        ("ka1.hermes.radio", &P_HERMES_RADIO),
-        ("ka2.hermes.radio", &P_HERMES_RADIO),
-        ("ka3.hermes.radio", &P_HERMES_RADIO),
-        ("ka4.hermes.radio", &P_HERMES_RADIO),
-        ("ka5.hermes.radio", &P_HERMES_RADIO),
-        ("ka6.hermes.radio", &P_HERMES_RADIO),
-        ("ka7.hermes.radio", &P_HERMES_RADIO),
-        ("ka8.hermes.radio", &P_HERMES_RADIO),
-        ("ka9.hermes.radio", &P_HERMES_RADIO),
-        ("ka10.hermes.radio", &P_HERMES_RADIO),
-        ("ka11.hermes.radio", &P_HERMES_RADIO),
-        ("ka12.hermes.radio", &P_HERMES_RADIO),
-        ("ka13.hermes.radio", &P_HERMES_RADIO),
-        ("ka14.hermes.radio", &P_HERMES_RADIO),
-        ("ka15.hermes.radio", &P_HERMES_RADIO),
-        ("ec.hermes.radio", &P_HERMES_RADIO),
-        ("ec1.hermes.radio", &P_HERMES_RADIO),
-        ("ec2.hermes.radio", &P_HERMES_RADIO),
-        ("ec3.hermes.radio", &P_HERMES_RADIO),
-        ("ec4.hermes.radio", &P_HERMES_RADIO),
-        ("ec5.hermes.radio", &P_HERMES_RADIO),
-        ("ec6.hermes.radio", &P_HERMES_RADIO),
-        ("ec7.hermes.radio", &P_HERMES_RADIO),
-        ("ec8.hermes.radio", &P_HERMES_RADIO),
-        ("ec9.hermes.radio", &P_HERMES_RADIO),
-        ("ec10.hermes.radio", &P_HERMES_RADIO),
-        ("ec11.hermes.radio", &P_HERMES_RADIO),
-        ("ec12.hermes.radio", &P_HERMES_RADIO),
-        ("ec13.hermes.radio", &P_HERMES_RADIO),
-        ("ec14.hermes.radio", &P_HERMES_RADIO),
-        ("ec15.hermes.radio", &P_HERMES_RADIO),
-        ("hermes.radio", &P_HERMES_RADIO),
-        ("hey.com", &P_HEY_COM),
-        ("i.ua", &P_I_UA),
-        ("i3.net", &P_I3_NET),
-        ("icloud.com", &P_ICLOUD),
-        ("me.com", &P_ICLOUD),
-        ("mac.com", &P_ICLOUD),
-        ("ik.me", &P_INFOMANIAK_COM),
-        ("kolst.com", &P_KOLST_COM),
-        ("kontent.com", &P_KONTENT_COM),
-        ("mail.de", &P_MAIL_DE),
-        ("mail.ru", &P_MAIL_RU),
-        ("inbox.ru", &P_MAIL_RU),
-        ("internet.ru", &P_MAIL_RU),
-        ("bk.ru", &P_MAIL_RU),
-        ("list.ru", &P_MAIL_RU),
-        ("mail2tor.com", &P_MAIL2TOR),
-        ("mailbox.org", &P_MAILBOX_ORG),
-        ("secure.mailbox.org", &P_MAILBOX_ORG),
-        ("mailo.com", &P_MAILO_COM),
-        ("nauta.cu", &P_NAUTA_CU),
-        ("naver.com", &P_NAVER),
-        ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
-        ("nubo.coop", &P_NUBO_COOP),
-        ("hotmail.com", &P_OUTLOOK_COM),
-        ("outlook.com", &P_OUTLOOK_COM),
-        ("office365.com", &P_OUTLOOK_COM),
-        ("outlook.com.tr", &P_OUTLOOK_COM),
-        ("live.com", &P_OUTLOOK_COM),
-        ("outlook.de", &P_OUTLOOK_COM),
-        ("ouvaton.org", &P_OUVATON_COOP),
-        ("posteo.de", &P_POSTEO),
-        ("posteo.af", &P_POSTEO),
-        ("posteo.at", &P_POSTEO),
-        ("posteo.be", &P_POSTEO),
-        ("posteo.ca", &P_POSTEO),
-        ("posteo.ch", &P_POSTEO),
-        ("posteo.cl", &P_POSTEO),
-        ("posteo.co", &P_POSTEO),
-        ("posteo.co.uk", &P_POSTEO),
-        ("posteo.com.br", &P_POSTEO),
-        ("posteo.cr", &P_POSTEO),
-        ("posteo.cz", &P_POSTEO),
-        ("posteo.dk", &P_POSTEO),
-        ("posteo.ee", &P_POSTEO),
-        ("posteo.es", &P_POSTEO),
-        ("posteo.eu", &P_POSTEO),
-        ("posteo.fi", &P_POSTEO),
-        ("posteo.gl", &P_POSTEO),
-        ("posteo.gr", &P_POSTEO),
-        ("posteo.hn", &P_POSTEO),
-        ("posteo.hr", &P_POSTEO),
-        ("posteo.hu", &P_POSTEO),
-        ("posteo.ie", &P_POSTEO),
-        ("posteo.in", &P_POSTEO),
-        ("posteo.is", &P_POSTEO),
-        ("posteo.it", &P_POSTEO),
-        ("posteo.jp", &P_POSTEO),
-        ("posteo.la", &P_POSTEO),
-        ("posteo.li", &P_POSTEO),
-        ("posteo.lt", &P_POSTEO),
-        ("posteo.lu", &P_POSTEO),
-        ("posteo.me", &P_POSTEO),
-        ("posteo.mx", &P_POSTEO),
-        ("posteo.my", &P_POSTEO),
-        ("posteo.net", &P_POSTEO),
-        ("posteo.nl", &P_POSTEO),
-        ("posteo.no", &P_POSTEO),
-        ("posteo.nz", &P_POSTEO),
-        ("posteo.org", &P_POSTEO),
-        ("posteo.pe", &P_POSTEO),
-        ("posteo.pl", &P_POSTEO),
-        ("posteo.pm", &P_POSTEO),
-        ("posteo.pt", &P_POSTEO),
-        ("posteo.ro", &P_POSTEO),
-        ("posteo.ru", &P_POSTEO),
-        ("posteo.se", &P_POSTEO),
-        ("posteo.sg", &P_POSTEO),
-        ("posteo.si", &P_POSTEO),
-        ("posteo.tn", &P_POSTEO),
-        ("posteo.uk", &P_POSTEO),
-        ("posteo.us", &P_POSTEO),
-        ("protonmail.com", &P_PROTONMAIL),
-        ("protonmail.ch", &P_PROTONMAIL),
-        ("pm.me", &P_PROTONMAIL),
-        ("qq.com", &P_QQ),
-        ("foxmail.com", &P_QQ),
-        ("riseup.net", &P_RISEUP_NET),
-        ("rogers.com", &P_ROGERS_COM),
-        ("sonic.net", &P_SONIC),
-        ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
-        ("solidaris.me", &P_SYSTEMAUSFALL_ORG),
-        ("systemli.org", &P_SYSTEMLI_ORG),
-        ("t-online.de", &P_T_ONLINE),
-        ("magenta.de", &P_T_ONLINE),
-        ("testrun.org", &P_TESTRUN),
-        ("tiscali.it", &P_TISCALI_IT),
-        ("tutanota.com", &P_TUTANOTA),
-        ("tutanota.de", &P_TUTANOTA),
-        ("tutamail.com", &P_TUTANOTA),
-        ("tuta.io", &P_TUTANOTA),
-        ("keemail.me", &P_TUTANOTA),
-        ("ukr.net", &P_UKR_NET),
-        ("undernet.uy", &P_UNDERNET_UY),
-        ("vfemail.net", &P_VFEMAIL),
-        ("vivaldi.net", &P_VIVALDI),
-        ("vodafone.de", &P_VODAFONE_DE),
-        ("vodafonemail.de", &P_VODAFONE_DE),
-        ("web.de", &P_WEB_DE),
-        ("email.de", &P_WEB_DE),
-        ("flirt.ms", &P_WEB_DE),
-        ("hallo.ms", &P_WEB_DE),
-        ("kuss.ms", &P_WEB_DE),
-        ("love.ms", &P_WEB_DE),
-        ("magic.ms", &P_WEB_DE),
-        ("singles.ms", &P_WEB_DE),
-        ("cool.ms", &P_WEB_DE),
-        ("kanzler.ms", &P_WEB_DE),
-        ("okay.ms", &P_WEB_DE),
-        ("party.ms", &P_WEB_DE),
-        ("pop.ms", &P_WEB_DE),
-        ("stars.ms", &P_WEB_DE),
-        ("techno.ms", &P_WEB_DE),
-        ("clever.ms", &P_WEB_DE),
-        ("deutschland.ms", &P_WEB_DE),
-        ("genial.ms", &P_WEB_DE),
-        ("ich.ms", &P_WEB_DE),
-        ("online.ms", &P_WEB_DE),
-        ("smart.ms", &P_WEB_DE),
-        ("wichtig.ms", &P_WEB_DE),
-        ("action.ms", &P_WEB_DE),
-        ("fussball.ms", &P_WEB_DE),
-        ("joker.ms", &P_WEB_DE),
-        ("planet.ms", &P_WEB_DE),
-        ("power.ms", &P_WEB_DE),
-        ("yahoo.com", &P_YAHOO),
-        ("yahoo.de", &P_YAHOO),
-        ("yahoo.it", &P_YAHOO),
-        ("yahoo.fr", &P_YAHOO),
-        ("yahoo.es", &P_YAHOO),
-        ("yahoo.se", &P_YAHOO),
-        ("yahoo.co.uk", &P_YAHOO),
-        ("yahoo.co.nz", &P_YAHOO),
-        ("yahoo.com.au", &P_YAHOO),
-        ("yahoo.com.ar", &P_YAHOO),
-        ("yahoo.com.br", &P_YAHOO),
-        ("yahoo.com.mx", &P_YAHOO),
-        ("ymail.com", &P_YAHOO),
-        ("rocketmail.com", &P_YAHOO),
-        ("yahoodns.net", &P_YAHOO),
-        ("yandex.com", &P_YANDEX_RU),
-        ("yandex.by", &P_YANDEX_RU),
-        ("yandex.kz", &P_YANDEX_RU),
-        ("yandex.ru", &P_YANDEX_RU),
-        ("yandex.ua", &P_YANDEX_RU),
-        ("ya.ru", &P_YANDEX_RU),
-        ("narod.ru", &P_YANDEX_RU),
-        ("yggmail", &P_YGGMAIL),
-        ("ziggo.nl", &P_ZIGGO_NL),
-        ("zohomail.eu", &P_ZOHO),
-        ("zohomail.com", &P_ZOHO),
-        ("zoho.com", &P_ZOHO),
-    ])
-});
+pub(crate) static PROVIDER_DATA: [(&str, &Provider); 318] = [
+    ("163.com", &P_163),
+    ("aktivix.org", &P_AKTIVIX_ORG),
+    ("aol.com", &P_AOL),
+    ("arcor.de", &P_ARCOR_DE),
+    ("autistici.org", &P_AUTISTICI_ORG),
+    ("delta.blinzeln.de", &P_BLINDZELN_ORG),
+    ("delta.blindzeln.org", &P_BLINDZELN_ORG),
+    ("bluewin.ch", &P_BLUEWIN_CH),
+    ("buzon.uy", &P_BUZON_UY),
+    ("c1.testrun.org", &P_C1_TESTRUN_ORG),
+    ("c2.testrun.org", &P_C2_TESTRUN_ORG),
+    ("c3.testrun.org", &P_C3_TESTRUN_ORG),
+    ("chello.at", &P_CHELLO_AT),
+    ("xfinity.com", &P_COMCAST),
+    ("comcast.net", &P_COMCAST),
+    ("dismail.de", &P_DISMAIL_DE),
+    ("disroot.org", &P_DISROOT),
+    ("e.email", &P_E_EMAIL),
+    ("espiv.net", &P_ESPIV_NET),
+    ("example.com", &P_EXAMPLE_COM),
+    ("example.org", &P_EXAMPLE_COM),
+    ("example.net", &P_EXAMPLE_COM),
+    ("123mail.org", &P_FASTMAIL),
+    ("150mail.com", &P_FASTMAIL),
+    ("150ml.com", &P_FASTMAIL),
+    ("16mail.com", &P_FASTMAIL),
+    ("2-mail.com", &P_FASTMAIL),
+    ("4email.net", &P_FASTMAIL),
+    ("50mail.com", &P_FASTMAIL),
+    ("airpost.net", &P_FASTMAIL),
+    ("allmail.net", &P_FASTMAIL),
+    ("bestmail.us", &P_FASTMAIL),
+    ("cluemail.com", &P_FASTMAIL),
+    ("elitemail.org", &P_FASTMAIL),
+    ("emailcorner.net", &P_FASTMAIL),
+    ("emailengine.net", &P_FASTMAIL),
+    ("emailengine.org", &P_FASTMAIL),
+    ("emailgroups.net", &P_FASTMAIL),
+    ("emailplus.org", &P_FASTMAIL),
+    ("emailuser.net", &P_FASTMAIL),
+    ("eml.cc", &P_FASTMAIL),
+    ("f-m.fm", &P_FASTMAIL),
+    ("fast-email.com", &P_FASTMAIL),
+    ("fast-mail.org", &P_FASTMAIL),
+    ("fastem.com", &P_FASTMAIL),
+    ("fastemail.us", &P_FASTMAIL),
+    ("fastemailer.com", &P_FASTMAIL),
+    ("fastest.cc", &P_FASTMAIL),
+    ("fastimap.com", &P_FASTMAIL),
+    ("fastmail.cn", &P_FASTMAIL),
+    ("fastmail.co.uk", &P_FASTMAIL),
+    ("fastmail.com", &P_FASTMAIL),
+    ("fastmail.com.au", &P_FASTMAIL),
+    ("fastmail.de", &P_FASTMAIL),
+    ("fastmail.es", &P_FASTMAIL),
+    ("fastmail.fm", &P_FASTMAIL),
+    ("fastmail.fr", &P_FASTMAIL),
+    ("fastmail.im", &P_FASTMAIL),
+    ("fastmail.in", &P_FASTMAIL),
+    ("fastmail.jp", &P_FASTMAIL),
+    ("fastmail.mx", &P_FASTMAIL),
+    ("fastmail.net", &P_FASTMAIL),
+    ("fastmail.nl", &P_FASTMAIL),
+    ("fastmail.org", &P_FASTMAIL),
+    ("fastmail.se", &P_FASTMAIL),
+    ("fastmail.to", &P_FASTMAIL),
+    ("fastmail.tw", &P_FASTMAIL),
+    ("fastmail.uk", &P_FASTMAIL),
+    ("fastmail.us", &P_FASTMAIL),
+    ("fastmailbox.net", &P_FASTMAIL),
+    ("fastmessaging.com", &P_FASTMAIL),
+    ("fea.st", &P_FASTMAIL),
+    ("fmail.co.uk", &P_FASTMAIL),
+    ("fmailbox.com", &P_FASTMAIL),
+    ("fmgirl.com", &P_FASTMAIL),
+    ("fmguy.com", &P_FASTMAIL),
+    ("ftml.net", &P_FASTMAIL),
+    ("h-mail.us", &P_FASTMAIL),
+    ("hailmail.net", &P_FASTMAIL),
+    ("imap-mail.com", &P_FASTMAIL),
+    ("imap.cc", &P_FASTMAIL),
+    ("imapmail.org", &P_FASTMAIL),
+    ("inoutbox.com", &P_FASTMAIL),
+    ("internet-e-mail.com", &P_FASTMAIL),
+    ("internet-mail.org", &P_FASTMAIL),
+    ("internetemails.net", &P_FASTMAIL),
+    ("internetmailing.net", &P_FASTMAIL),
+    ("jetemail.net", &P_FASTMAIL),
+    ("justemail.net", &P_FASTMAIL),
+    ("letterboxes.org", &P_FASTMAIL),
+    ("mail-central.com", &P_FASTMAIL),
+    ("mail-page.com", &P_FASTMAIL),
+    ("mailandftp.com", &P_FASTMAIL),
+    ("mailas.com", &P_FASTMAIL),
+    ("mailbolt.com", &P_FASTMAIL),
+    ("mailc.net", &P_FASTMAIL),
+    ("mailcan.com", &P_FASTMAIL),
+    ("mailforce.net", &P_FASTMAIL),
+    ("mailftp.com", &P_FASTMAIL),
+    ("mailhaven.com", &P_FASTMAIL),
+    ("mailingaddress.org", &P_FASTMAIL),
+    ("mailite.com", &P_FASTMAIL),
+    ("mailmight.com", &P_FASTMAIL),
+    ("mailnew.com", &P_FASTMAIL),
+    ("mailsent.net", &P_FASTMAIL),
+    ("mailservice.ms", &P_FASTMAIL),
+    ("mailup.net", &P_FASTMAIL),
+    ("mailworks.org", &P_FASTMAIL),
+    ("ml1.net", &P_FASTMAIL),
+    ("mm.st", &P_FASTMAIL),
+    ("myfastmail.com", &P_FASTMAIL),
+    ("mymacmail.com", &P_FASTMAIL),
+    ("nospammail.net", &P_FASTMAIL),
+    ("ownmail.net", &P_FASTMAIL),
+    ("petml.com", &P_FASTMAIL),
+    ("postinbox.com", &P_FASTMAIL),
+    ("postpro.net", &P_FASTMAIL),
+    ("proinbox.com", &P_FASTMAIL),
+    ("promessage.com", &P_FASTMAIL),
+    ("realemail.net", &P_FASTMAIL),
+    ("reallyfast.biz", &P_FASTMAIL),
+    ("reallyfast.info", &P_FASTMAIL),
+    ("rushpost.com", &P_FASTMAIL),
+    ("sent.as", &P_FASTMAIL),
+    ("sent.at", &P_FASTMAIL),
+    ("sent.com", &P_FASTMAIL),
+    ("speedpost.net", &P_FASTMAIL),
+    ("speedymail.org", &P_FASTMAIL),
+    ("ssl-mail.com", &P_FASTMAIL),
+    ("swift-mail.com", &P_FASTMAIL),
+    ("the-fastest.net", &P_FASTMAIL),
+    ("the-quickest.com", &P_FASTMAIL),
+    ("theinternetemail.com", &P_FASTMAIL),
+    ("veryfast.biz", &P_FASTMAIL),
+    ("veryspeedy.net", &P_FASTMAIL),
+    ("warpmail.net", &P_FASTMAIL),
+    ("xsmail.com", &P_FASTMAIL),
+    ("yepmail.net", &P_FASTMAIL),
+    ("your-mail.com", &P_FASTMAIL),
+    ("firemail.at", &P_FIREMAIL_DE),
+    ("firemail.de", &P_FIREMAIL_DE),
+    ("five.chat", &P_FIVE_CHAT),
+    ("freenet.de", &P_FREENET_DE),
+    ("gmail.com", &P_GMAIL),
+    ("googlemail.com", &P_GMAIL),
+    ("google.com", &P_GMAIL),
+    ("gmx.net", &P_GMX_NET),
+    ("gmx.de", &P_GMX_NET),
+    ("gmx.at", &P_GMX_NET),
+    ("gmx.ch", &P_GMX_NET),
+    ("gmx.org", &P_GMX_NET),
+    ("gmx.eu", &P_GMX_NET),
+    ("gmx.info", &P_GMX_NET),
+    ("gmx.biz", &P_GMX_NET),
+    ("gmx.com", &P_GMX_NET),
+    ("*.hermes.radio", &P_HERMES_RADIO),
+    ("*.aco-connexion.org", &P_HERMES_RADIO),
+    ("hey.com", &P_HEY_COM),
+    ("i.ua", &P_I_UA),
+    ("i3.net", &P_I3_NET),
+    ("icloud.com", &P_ICLOUD),
+    ("me.com", &P_ICLOUD),
+    ("mac.com", &P_ICLOUD),
+    ("ik.me", &P_INFOMANIAK_COM),
+    ("kolst.com", &P_KOLST_COM),
+    ("kontent.com", &P_KONTENT_COM),
+    ("mail.de", &P_MAIL_DE),
+    ("mail.ru", &P_MAIL_RU),
+    ("inbox.ru", &P_MAIL_RU),
+    ("internet.ru", &P_MAIL_RU),
+    ("bk.ru", &P_MAIL_RU),
+    ("list.ru", &P_MAIL_RU),
+    ("mail2tor.com", &P_MAIL2TOR),
+    ("mailbox.org", &P_MAILBOX_ORG),
+    ("secure.mailbox.org", &P_MAILBOX_ORG),
+    ("mailo.com", &P_MAILO_COM),
+    ("nauta.cu", &P_NAUTA_CU),
+    ("naver.com", &P_NAVER),
+    ("nine.testrun.org", &P_NINE_TESTRUN_ORG),
+    ("nubo.coop", &P_NUBO_COOP),
+    ("hotmail.com", &P_OUTLOOK_COM),
+    ("outlook.com", &P_OUTLOOK_COM),
+    ("office365.com", &P_OUTLOOK_COM),
+    ("outlook.com.tr", &P_OUTLOOK_COM),
+    ("live.com", &P_OUTLOOK_COM),
+    ("outlook.de", &P_OUTLOOK_COM),
+    ("ouvaton.org", &P_OUVATON_COOP),
+    ("posteo.de", &P_POSTEO),
+    ("posteo.af", &P_POSTEO),
+    ("posteo.at", &P_POSTEO),
+    ("posteo.be", &P_POSTEO),
+    ("posteo.ca", &P_POSTEO),
+    ("posteo.ch", &P_POSTEO),
+    ("posteo.cl", &P_POSTEO),
+    ("posteo.co", &P_POSTEO),
+    ("posteo.co.uk", &P_POSTEO),
+    ("posteo.com.br", &P_POSTEO),
+    ("posteo.cr", &P_POSTEO),
+    ("posteo.cz", &P_POSTEO),
+    ("posteo.dk", &P_POSTEO),
+    ("posteo.ee", &P_POSTEO),
+    ("posteo.es", &P_POSTEO),
+    ("posteo.eu", &P_POSTEO),
+    ("posteo.fi", &P_POSTEO),
+    ("posteo.gl", &P_POSTEO),
+    ("posteo.gr", &P_POSTEO),
+    ("posteo.hn", &P_POSTEO),
+    ("posteo.hr", &P_POSTEO),
+    ("posteo.hu", &P_POSTEO),
+    ("posteo.ie", &P_POSTEO),
+    ("posteo.in", &P_POSTEO),
+    ("posteo.is", &P_POSTEO),
+    ("posteo.it", &P_POSTEO),
+    ("posteo.jp", &P_POSTEO),
+    ("posteo.la", &P_POSTEO),
+    ("posteo.li", &P_POSTEO),
+    ("posteo.lt", &P_POSTEO),
+    ("posteo.lu", &P_POSTEO),
+    ("posteo.me", &P_POSTEO),
+    ("posteo.mx", &P_POSTEO),
+    ("posteo.my", &P_POSTEO),
+    ("posteo.net", &P_POSTEO),
+    ("posteo.nl", &P_POSTEO),
+    ("posteo.no", &P_POSTEO),
+    ("posteo.nz", &P_POSTEO),
+    ("posteo.org", &P_POSTEO),
+    ("posteo.pe", &P_POSTEO),
+    ("posteo.pl", &P_POSTEO),
+    ("posteo.pm", &P_POSTEO),
+    ("posteo.pt", &P_POSTEO),
+    ("posteo.ro", &P_POSTEO),
+    ("posteo.ru", &P_POSTEO),
+    ("posteo.se", &P_POSTEO),
+    ("posteo.sg", &P_POSTEO),
+    ("posteo.si", &P_POSTEO),
+    ("posteo.tn", &P_POSTEO),
+    ("posteo.uk", &P_POSTEO),
+    ("posteo.us", &P_POSTEO),
+    ("protonmail.com", &P_PROTONMAIL),
+    ("protonmail.ch", &P_PROTONMAIL),
+    ("pm.me", &P_PROTONMAIL),
+    ("qq.com", &P_QQ),
+    ("foxmail.com", &P_QQ),
+    ("riseup.net", &P_RISEUP_NET),
+    ("rogers.com", &P_ROGERS_COM),
+    ("sonic.net", &P_SONIC),
+    ("systemausfall.org", &P_SYSTEMAUSFALL_ORG),
+    ("solidaris.me", &P_SYSTEMAUSFALL_ORG),
+    ("systemli.org", &P_SYSTEMLI_ORG),
+    ("t-online.de", &P_T_ONLINE),
+    ("magenta.de", &P_T_ONLINE),
+    ("testrun.org", &P_TESTRUN),
+    ("tiscali.it", &P_TISCALI_IT),
+    ("tutanota.com", &P_TUTANOTA),
+    ("tutanota.de", &P_TUTANOTA),
+    ("tutamail.com", &P_TUTANOTA),
+    ("tuta.io", &P_TUTANOTA),
+    ("keemail.me", &P_TUTANOTA),
+    ("ukr.net", &P_UKR_NET),
+    ("undernet.uy", &P_UNDERNET_UY),
+    ("vfemail.net", &P_VFEMAIL),
+    ("vivaldi.net", &P_VIVALDI),
+    ("vodafone.de", &P_VODAFONE_DE),
+    ("vodafonemail.de", &P_VODAFONE_DE),
+    ("web.de", &P_WEB_DE),
+    ("email.de", &P_WEB_DE),
+    ("flirt.ms", &P_WEB_DE),
+    ("hallo.ms", &P_WEB_DE),
+    ("kuss.ms", &P_WEB_DE),
+    ("love.ms", &P_WEB_DE),
+    ("magic.ms", &P_WEB_DE),
+    ("singles.ms", &P_WEB_DE),
+    ("cool.ms", &P_WEB_DE),
+    ("kanzler.ms", &P_WEB_DE),
+    ("okay.ms", &P_WEB_DE),
+    ("party.ms", &P_WEB_DE),
+    ("pop.ms", &P_WEB_DE),
+    ("stars.ms", &P_WEB_DE),
+    ("techno.ms", &P_WEB_DE),
+    ("clever.ms", &P_WEB_DE),
+    ("deutschland.ms", &P_WEB_DE),
+    ("genial.ms", &P_WEB_DE),
+    ("ich.ms", &P_WEB_DE),
+    ("online.ms", &P_WEB_DE),
+    ("smart.ms", &P_WEB_DE),
+    ("wichtig.ms", &P_WEB_DE),
+    ("action.ms", &P_WEB_DE),
+    ("fussball.ms", &P_WEB_DE),
+    ("joker.ms", &P_WEB_DE),
+    ("planet.ms", &P_WEB_DE),
+    ("power.ms", &P_WEB_DE),
+    ("yahoo.com", &P_YAHOO),
+    ("yahoo.de", &P_YAHOO),
+    ("yahoo.it", &P_YAHOO),
+    ("yahoo.fr", &P_YAHOO),
+    ("yahoo.es", &P_YAHOO),
+    ("yahoo.se", &P_YAHOO),
+    ("yahoo.co.uk", &P_YAHOO),
+    ("yahoo.co.nz", &P_YAHOO),
+    ("yahoo.com.au", &P_YAHOO),
+    ("yahoo.com.ar", &P_YAHOO),
+    ("yahoo.com.br", &P_YAHOO),
+    ("yahoo.com.mx", &P_YAHOO),
+    ("ymail.com", &P_YAHOO),
+    ("rocketmail.com", &P_YAHOO),
+    ("yahoodns.net", &P_YAHOO),
+    ("yandex.com", &P_YANDEX_RU),
+    ("yandex.by", &P_YANDEX_RU),
+    ("yandex.kz", &P_YANDEX_RU),
+    ("yandex.ru", &P_YANDEX_RU),
+    ("yandex.ua", &P_YANDEX_RU),
+    ("ya.ru", &P_YANDEX_RU),
+    ("narod.ru", &P_YANDEX_RU),
+    ("yggmail", &P_YGGMAIL),
+    ("ziggo.nl", &P_ZIGGO_NL),
+    ("zohomail.eu", &P_ZOHO),
+    ("zohomail.com", &P_ZOHO),
+    ("zoho.com", &P_ZOHO),
+];
 
 pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     HashMap::from([
@@ -2050,4 +2001,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static _PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2023, 11, 5).unwrap());
+    Lazy::new(|| chrono::NaiveDate::from_ymd_opt(2024, 2, 5).unwrap());

src/push.rs

@@ -0,0 +1,115 @@
+use std::sync::atomic::Ordering;
+use std::sync::Arc;
+
+use anyhow::Result;
+use tokio::sync::RwLock;
+
+use crate::context::Context;
+use crate::net::http;
+
+/// Manages subscription to Apple Push Notification services.
+///
+/// This structure is created by account manager and is shared between accounts.
+/// To enable notifications, application should request the device token as described in
+/// <https://developer.apple.com/documentation/usernotifications/registering-your-app-with-apns>
+/// and give it to the account manager, which will forward the token in this structure.
+///
+/// Each account (context) can then retrieve device token
+/// from this structure and give it to the email server.
+/// If email server does not support push notifications,
+/// account can call `subscribe` method
+/// to register device token with the heartbeat
+/// notification provider server as a fallback.
+#[derive(Debug, Clone, Default)]
+pub struct PushSubscriber {
+    inner: Arc<RwLock<PushSubscriberState>>,
+}
+
+impl PushSubscriber {
+    /// Creates new push notification subscriber.
+    pub(crate) fn new() -> Self {
+        Default::default()
+    }
+
+    /// Sets device token for Apple Push Notification service.
+    pub(crate) async fn set_device_token(&mut self, token: &str) {
+        self.inner.write().await.device_token = Some(token.to_string());
+    }
+
+    /// Retrieves device token.
+    ///
+    /// Token may be not available if application is not running on Apple platform,
+    /// failed to register for remote notifications or is in the process of registering.
+    ///
+    /// IMAP loop should periodically check if device token is available
+    /// and send the token to the email server if it supports push notifications.
+    pub(crate) async fn device_token(&self) -> Option<String> {
+        self.inner.read().await.device_token.clone()
+    }
+
+    /// Subscribes for heartbeat notifications with previously set device token.
+    pub(crate) async fn subscribe(&self) -> Result<()> {
+        let mut state = self.inner.write().await;
+
+        if state.heartbeat_subscribed {
+            return Ok(());
+        }
+
+        let Some(ref token) = state.device_token else {
+            return Ok(());
+        };
+
+        let socks5_config = None;
+        let response = http::get_client(socks5_config)?
+            .post("https://notifications.delta.chat/register")
+            .body(format!("{{\"token\":\"{token}\"}}"))
+            .send()
+            .await?;
+
+        let response_status = response.status();
+        if response_status.is_success() {
+            state.heartbeat_subscribed = true;
+        }
+        Ok(())
+    }
+
+    pub(crate) async fn heartbeat_subscribed(&self) -> bool {
+        self.inner.read().await.heartbeat_subscribed
+    }
+}
+
+#[derive(Debug, Default)]
+pub(crate) struct PushSubscriberState {
+    /// Device token.
+    device_token: Option<String>,
+
+    /// If subscribed to heartbeat push notifications.
+    heartbeat_subscribed: bool,
+}
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[repr(i8)]
+pub enum NotifyState {
+    /// Not subscribed to push notifications.
+    #[default]
+    NotConnected = 0,
+
+    /// Subscribed to heartbeat push notifications.
+    Heartbeat = 1,
+
+    /// Subscribed to push notifications for new messages.
+    Connected = 2,
+}
+
+impl Context {
+    /// Returns push notification subscriber state.
+    pub async fn push_state(&self) -> NotifyState {
+        if self.push_subscribed.load(Ordering::Relaxed) {
+            NotifyState::Connected
+        } else if self.push_subscriber.heartbeat_subscribed().await {
+            NotifyState::Heartbeat
+        } else {
+            NotifyState::NotConnected
+        }
+    }
+}

src/qr.rs

@@ -23,8 +23,11 @@ use crate::message::Message;
 use crate::peerstate::Peerstate;
 use crate::socks::Socks5Config;
 use crate::token;
+use crate::tools::validate_id;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
+const IDELTACHAT_SCHEME: &str = "https://i.delta.chat/#";
+const IDELTACHAT_NOSLASH_SCHEME: &str = "https://i.delta.chat#";
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
 pub(super) const DCLOGIN_SCHEME: &str = "DCLOGIN:";
 const DCWEBRTC_SCHEME: &str = "DCWEBRTC:";
@@ -247,12 +250,14 @@ fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
 pub async fn check_qr(context: &Context, qr: &str) -> Result<Qr> {
-    info!(context, "Scanned QR code: {}", qr);
-
     let qrcode = if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
         decode_openpgp(context, qr)
             .await
             .context("failed to decode OPENPGP4FPR QR code")?
+    } else if qr.starts_with(IDELTACHAT_SCHEME) {
+        decode_ideltachat(context, IDELTACHAT_SCHEME, qr).await?
+    } else if qr.starts_with(IDELTACHAT_NOSLASH_SCHEME) {
+        decode_ideltachat(context, IDELTACHAT_NOSLASH_SCHEME, qr).await?
     } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
         decode_account(qr)?
     } else if starts_with_ignore_case(qr, DCLOGIN_SCHEME) {
@@ -301,11 +306,12 @@ pub fn format_backup(qr: &Qr) -> Result<String> {
 async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     let payload = &qr[OPENPGP4FPR_SCHEME.len()..];
 
-    let (fingerprint, fragment) = match payload.find('#').map(|offset| {
-        let (fp, rest) = payload.split_at(offset);
-        // need to remove the # from the fragment
-        (fp, &rest[1..])
-    }) {
+    // macOS and iOS sometimes replace the # with %23 (uri encode it), we should be able to parse this wrong format too.
+    // see issue https://github.com/deltachat/deltachat-core-rust/issues/1969 for more info
+    let (fingerprint, fragment) = match payload
+        .split_once('#')
+        .or_else(|| payload.split_once("%23"))
+    {
         Some(pair) => pair,
         None => (payload, ""),
     };
@@ -340,9 +346,18 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
         "".to_string()
     };
 
-    let invitenumber = param.get("i").map(|s| s.to_string());
-    let authcode = param.get("s").map(|s| s.to_string());
-    let grpid = param.get("x").map(|s| s.to_string());
+    let invitenumber = param
+        .get("i")
+        .filter(|&s| validate_id(s))
+        .map(|s| s.to_string());
+    let authcode = param
+        .get("s")
+        .filter(|&s| validate_id(s))
+        .map(|s| s.to_string());
+    let grpid = param
+        .get("x")
+        .filter(|&s| validate_id(s))
+        .map(|s| s.to_string());
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get("g") {
@@ -376,7 +391,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
                 .await
                 .with_context(|| format!("can't check if address {addr:?} is our address"))?
             {
-                if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await {
+                if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await? {
                     Ok(Qr::WithdrawVerifyGroup {
                         grpname,
                         grpid,
@@ -406,7 +421,7 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
                 })
             }
         } else if context.is_self_addr(&addr).await? {
-            if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await {
+            if token::exists(context, token::Namespace::InviteNumber, &invitenumber).await? {
                 Ok(Qr::WithdrawVerifyContact {
                     contact_id,
                     fingerprint,
@@ -453,13 +468,21 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Result<Qr> {
     }
 }
 
+/// scheme: `https://i.delta.chat[/]#FINGERPRINT&a=ADDR[&OPTIONAL_PARAMS]`
+async fn decode_ideltachat(context: &Context, prefix: &str, qr: &str) -> Result<Qr> {
+    let qr = qr.replacen(prefix, OPENPGP4FPR_SCHEME, 1);
+    let qr = qr.replacen('&', "#", 1);
+    decode_openpgp(context, &qr)
+        .await
+        .context("failed to decode {prefix} QR code")
+}
+
 /// scheme: `DCACCOUNT:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3`
 fn decode_account(qr: &str) -> Result<Qr> {
     let payload = qr
         .get(DCACCOUNT_SCHEME.len()..)
         .context("invalid DCACCOUNT payload")?;
-    let url =
-        url::Url::parse(payload).with_context(|| format!("Invalid account URL: {payload:?}"))?;
+    let url = url::Url::parse(payload).context("Invalid account URL")?;
     if url.scheme() == "http" || url.scheme() == "https" {
         Ok(Qr::Account {
             domain: url
@@ -468,7 +491,7 @@ fn decode_account(qr: &str) -> Result<Qr> {
                 .to_string(),
         })
     } else {
-        bail!("Bad scheme for account URL: {:?}.", payload);
+        bail!("Bad scheme for account URL: {:?}.", url.scheme());
     }
 }
 
@@ -479,8 +502,7 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result<Qr> {
         .context("invalid DCWEBRTC payload")?;
 
     let (_type, url) = Message::parse_webrtc_instance(payload);
-    let url =
-        url::Url::parse(&url).with_context(|| format!("Invalid WebRTC instance: {payload:?}"))?;
+    let url = url::Url::parse(&url).context("Invalid WebRTC instance")?;
 
     if url.scheme() == "http" || url.scheme() == "https" {
         Ok(Qr::WebrtcInstance {
@@ -491,7 +513,7 @@ fn decode_webrtc_instance(_context: &Context, qr: &str) -> Result<Qr> {
             instance_pattern: payload.to_string(),
         })
     } else {
-        bail!("Bad URL scheme for WebRTC instance: {:?}", payload);
+        bail!("Bad URL scheme for WebRTC instance: {:?}", url.scheme());
     }
 }
 
@@ -533,19 +555,22 @@ async fn set_account_from_qr(context: &Context, qr: &str) -> Result<()> {
         .send()
         .await?;
     let response_status = response.status();
-    let response_text = response.text().await.with_context(|| {
-        format!("Cannot create account, request to {url_str:?} failed: empty response")
-    })?;
+    let response_text = response
+        .text()
+        .await
+        .context("Cannot create account, request failed: empty response")?;
 
     if response_status.is_success() {
         let CreateAccountSuccessResponse { password, email } = serde_json::from_str(&response_text)
             .with_context(|| {
-                format!(
-                    "Cannot create account, response from {url_str:?} is malformed:\n{response_text:?}"
-                )
+                format!("Cannot create account, response is malformed:\n{response_text:?}")
             })?;
-        context.set_config(Config::Addr, Some(&email)).await?;
-        context.set_config(Config::MailPw, Some(&password)).await?;
+        context
+            .set_config_internal(Config::Addr, Some(&email))
+            .await?;
+        context
+            .set_config_internal(Config::MailPw, Some(&password))
+            .await?;
 
         Ok(())
     } else {
@@ -573,7 +598,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
             instance_pattern,
         } => {
             context
-                .set_config(Config::WebrtcInstance, Some(&instance_pattern))
+                .set_config_internal(Config::WebrtcInstance, Some(&instance_pattern))
                 .await?;
         }
         Qr::WithdrawVerifyContact {
@@ -633,7 +658,7 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<()> {
         Qr::Login { address, options } => {
             configure_from_login_qr(context, &address, options).await?
         }
-        _ => bail!("qr code {:?} does not contain config", qr),
+        _ => bail!("QR code does not contain config"),
     }
 
     Ok(())
@@ -943,6 +968,40 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decode_ideltachat_link() -> Result<()> {
+        let ctx = TestContext::new().await;
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "https://i.delta.chat/#79252762C34C5096AF57958F4FC3D21A81B0F0A7&a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "https://i.delta.chat#79252762C34C5096AF57958F4FC3D21A81B0F0A7&a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+
+        Ok(())
+    }
+
+    // macOS and iOS sometimes replace the # with %23 (uri encode it), we should be able to parse this wrong format too.
+    // see issue https://github.com/deltachat/deltachat-core-rust/issues/1969 for more info
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decode_openpgp_tolerance_for_issue_1969() -> Result<()> {
+        let ctx = TestContext::new().await;
+
+        let qr = check_qr(
+            &ctx.ctx,
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7%23a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+        ).await?;
+
+        assert!(matches!(qr, Qr::AskVerifyGroup { .. }));
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_decode_openpgp_group() -> Result<()> {
         let ctx = TestContext::new().await;
@@ -986,6 +1045,21 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_decode_openpgp_invalid_token() -> Result<()> {
+        let ctx = TestContext::new().await;
+
+        // Token cannot contain "/"
+        let qr = check_qr(
+            &ctx.ctx,
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL/cxRL"
+        ).await?;
+
+        assert!(matches!(qr, Qr::FprMismatch { .. }));
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_decode_openpgp_secure_join() -> Result<()> {
         let ctx = TestContext::new().await;

src/qr/dclogin_scheme.rs

@@ -163,7 +163,9 @@ pub(crate) async fn configure_from_login_qr(
     address: &str,
     options: LoginOptions,
 ) -> Result<()> {
-    context.set_config(Config::Addr, Some(address)).await?;
+    context
+        .set_config_internal(Config::Addr, Some(address))
+        .await?;
 
     match options {
         LoginOptions::V1 {
@@ -181,27 +183,35 @@ pub(crate) async fn configure_from_login_qr(
             smtp_security,
             smtp_certificate_checks,
         } => {
-            context.set_config(Config::MailPw, Some(&mail_pw)).await?;
+            context
+                .set_config_internal(Config::MailPw, Some(&mail_pw))
+                .await?;
             if let Some(value) = imap_host {
-                context.set_config(Config::MailServer, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailServer, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_port {
                 context
-                    .set_config(Config::MailPort, Some(&value.to_string()))
+                    .set_config_internal(Config::MailPort, Some(&value.to_string()))
                     .await?;
             }
             if let Some(value) = imap_username {
-                context.set_config(Config::MailUser, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailUser, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_password {
-                context.set_config(Config::MailPw, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::MailPw, Some(&value))
+                    .await?;
             }
             if let Some(value) = imap_security {
                 let code = value
                     .to_u8()
                     .context("could not convert imap security value to number")?;
                 context
-                    .set_config(Config::MailSecurity, Some(&code.to_string()))
+                    .set_config_internal(Config::MailSecurity, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = imap_certificate_checks {
@@ -209,29 +219,35 @@ pub(crate) async fn configure_from_login_qr(
                     .to_u32()
                     .context("could not convert imap certificate checks value to number")?;
                 context
-                    .set_config(Config::ImapCertificateChecks, Some(&code.to_string()))
+                    .set_config_internal(Config::ImapCertificateChecks, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_host {
-                context.set_config(Config::SendServer, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendServer, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_port {
                 context
-                    .set_config(Config::SendPort, Some(&value.to_string()))
+                    .set_config_internal(Config::SendPort, Some(&value.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_username {
-                context.set_config(Config::SendUser, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendUser, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_password {
-                context.set_config(Config::SendPw, Some(&value)).await?;
+                context
+                    .set_config_internal(Config::SendPw, Some(&value))
+                    .await?;
             }
             if let Some(value) = smtp_security {
                 let code = value
                     .to_u8()
                     .context("could not convert smtp security value to number")?;
                 context
-                    .set_config(Config::SendSecurity, Some(&code.to_string()))
+                    .set_config_internal(Config::SendSecurity, Some(&code.to_string()))
                     .await?;
             }
             if let Some(value) = smtp_certificate_checks {
@@ -239,7 +255,7 @@ pub(crate) async fn configure_from_login_qr(
                     .to_u32()
                     .context("could not convert smtp certificate checks value to number")?;
                 context
-                    .set_config(Config::SmtpCertificateChecks, Some(&code.to_string()))
+                    .set_config_internal(Config::SmtpCertificateChecks, Some(&code.to_string()))
                     .await?;
             }
             Ok(())

src/quota.rs

@@ -10,9 +10,8 @@ use crate::config::Config;
 use crate::context::Context;
 use crate::imap::scan_folders::get_watched_folders;
 use crate::imap::session::Session as ImapSession;
-use crate::imap::Imap;
 use crate::message::{Message, Viewtype};
-use crate::tools::time;
+use crate::tools;
 use crate::{stock_str, EventType};
 
 /// warn about a nearly full mailbox after this usage percentage is reached.
@@ -40,8 +39,8 @@ pub struct QuotaInfo {
     /// set to `Ok()` for valid quota information.
     pub(crate) recent: Result<BTreeMap<String, Vec<QuotaResource>>>,
 
-    /// Timestamp when structure was modified.
-    pub(crate) modified: i64,
+    /// When the structure was modified.
+    pub(crate) modified: tools::Time,
 }
 
 async fn get_unique_quota_roots_and_usage(
@@ -111,13 +110,7 @@ impl Context {
     /// As the message is added only once, the user is not spammed
     /// in case for some providers the quota is always at ~100%
     /// and new space is allocated as needed.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<()> {
-        if let Err(err) = imap.prepare(self).await {
-            warn!(self, "could not connect: {:#}", err);
-            return Ok(());
-        }
-
-        let session = imap.session.as_mut().context("no session")?;
+    pub(crate) async fn update_recent_quota(&self, session: &mut ImapSession) -> Result<()> {
         let quota = if session.can_check_quota() {
             let folders = get_watched_folders(self).await?;
             get_unique_quota_roots_and_usage(session, folders).await
@@ -132,13 +125,17 @@ impl Context {
                         highest,
                         self.get_config_int(Config::QuotaExceeding).await? as u64,
                     ) {
-                        self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
-                            .await?;
+                        self.set_config_internal(
+                            Config::QuotaExceeding,
+                            Some(&highest.to_string()),
+                        )
+                        .await?;
                         let mut msg = Message::new(Viewtype::Text);
                         msg.text = stock_str::quota_exceeding(self, highest).await;
                         add_device_msg_with_importance(self, None, Some(&mut msg), true).await?;
                     } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
-                        self.set_config(Config::QuotaExceeding, None).await?;
+                        self.set_config_internal(Config::QuotaExceeding, None)
+                            .await?;
                     }
                 }
                 Err(err) => warn!(self, "cannot get highest quota usage: {:#}", err),
@@ -147,7 +144,7 @@ impl Context {
 
         *self.quota.write().await = Some(QuotaInfo {
             recent: quota,
-            modified: time(),
+            modified: tools::Time::now(),
         });
 
         self.emit_event(EventType::ConnectivityChanged);