Test windows with Rust 1.77

This reverts commit d5af7b05d0.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.77.1
+      RUSTUP_TOOLCHAIN: 1.78.0
     steps:
       - uses: actions/checkout@v4
         with:
@@ -83,11 +83,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.77.1
+            rust: 1.78.0
           - os: windows-latest
-            rust: 1.77.1
+            rust: 1.77.0
           - os: macos-latest
-            rust: 1.77.1
+            rust: 1.78.0
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest

.github/workflows/deltachat-rpc-server.yml

@@ -266,3 +266,128 @@ jobs:
       - name: Publish deltachat-rpc-client to PyPI
         if: github.event_name == 'release'
         uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish_npm_package:
+    name: Build & Publish npm prebuilds and deltachat-rpc-server
+    needs: ["build_linux", "build_windows", "build_macos"]
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11' 
+
+      - name: Download Linux aarch64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-aarch64-linux
+          path: deltachat-rpc-server-aarch64-linux.d
+
+      - name: Download Linux armv7l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv7l-linux
+          path: deltachat-rpc-server-armv7l-linux.d
+
+      - name: Download Linux armv6l binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armv6l-linux
+          path: deltachat-rpc-server-armv6l-linux.d
+
+      - name: Download Linux i686 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-i686-linux
+          path: deltachat-rpc-server-i686-linux.d
+
+      - name: Download Linux x86_64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-linux
+          path: deltachat-rpc-server-x86_64-linux.d
+
+      - name: Download Win32 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win32
+          path: deltachat-rpc-server-win32.d
+
+      - name: Download Win64 binary
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-win64
+          path: deltachat-rpc-server-win64.d
+
+      - name: Download macOS binary for x86_64
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-x86_64-macos
+          path: deltachat-rpc-server-x86_64-macos.d
+
+      - name: Download macOS binary for aarch64
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-aarch64-macos
+          path: deltachat-rpc-server-aarch64-macos.d
+
+      - name: Download Android binary for arm64-v8a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-arm64-v8a-android
+          path: deltachat-rpc-server-arm64-v8a-android.d
+
+      - name: Download Android binary for armeabi-v7a
+        uses: actions/download-artifact@v4
+        with:
+          name: deltachat-rpc-server-armeabi-v7a-android
+          path: deltachat-rpc-server-armeabi-v7a-android.d
+
+      - name: make npm packets for prebuilds and `@deltachat/stdio-rpc-server`
+        run: |
+          cd deltachat-rpc-server/npm-package
+
+          python --version
+
+          python scripts/pack_binary_for_platform.py aarch64-unknown-linux-musl     ../../deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py armv7-unknown-linux-musleabihf ../../deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py arm-unknown-linux-musleabihf   ../../deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py i686-unknown-linux-musl        ../../deltachat-rpc-server-i686-linux.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py x86_64-unknown-linux-musl      ../../deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py i686-pc-windows-gnu            ../../deltachat-rpc-server-win32.d/deltachat-rpc-server.exe 
+          python scripts/pack_binary_for_platform.py x86_64-pc-windows-gnu          ../../deltachat-rpc-server-win64.d/deltachat-rpc-server.exe 
+          python scripts/pack_binary_for_platform.py x86_64-apple-darwin            ../../deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py aarch64-apple-darwin           ../../deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server 
+          python scripts/pack_binary_for_platform.py aarch64-linux-android          ../../deltachat-rpc-server-arm64-v8a-android.d/deltachat-rpc-server
+          python scripts/pack_binary_for_platform.py armv7-linux-androideabi        ../../deltachat-rpc-server-armeabi-v7a-android.d/deltachat-rpc-server
+
+          ls -lah platform_package
+
+          for platform in ./platform_package/*; do npm pack "$platform"; done
+          npm pack    
+          ls -lah
+      
+      - name: Upload to artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: deltachat-rpc-server-npm-package
+          path: deltachat-rpc-server/npm-package/*.tgz
+          if-no-files-found: error
+
+      - name: Upload npm packets to the GitHub release
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+        run: |
+          gh release upload ${{ github.ref_name }} \
+            --repo ${{ github.repository }} \
+            *.tgz
+
+      - name: Publish npm packets for prebuilds and `@deltachat/stdio-rpc-server`
+        if: github.event_name == 'release'
+        run: |
+          ls -lah platform_package
+
+          for platform in *.tgz; do npm publish "$platform"; done

.github/workflows/upload-docs.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - build_jsonrpc_docs_ci
 
 jobs:
   build-rs:
@@ -17,13 +18,11 @@ jobs:
         run: |
           cargo doc --package deltachat --no-deps --document-private-items
       - name: Upload to rs.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: ${{ secrets.USERNAME }}
-          KEY: ${{ secrets.KEY }}
-          HOST: "delta.chat"
-          SOURCE: "target/doc"
-          TARGET: "/var/www/html/rs/"
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
 
   build-python:
     runs-on: ubuntu-latest
@@ -62,3 +61,31 @@ jobs:
           echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
           rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
+
+  build-ts:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./deltachat-jsonrpc/typescript
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - name: Use Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+      - name: npm install
+        run: npm install
+      - name: npm run build
+        run: npm run build
+      - name: Run docs script
+        run: npm run docs
+      - name: Upload to js.jsonrpc.delta.chat
+        run: |
+          mkdir -p "$HOME/.ssh"
+          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
+          chmod 600 "$HOME/.ssh/key"
+          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"

.gitignore

@@ -33,6 +33,7 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
+.vscode
 .vscode/launch.json
 python/accounts.txt
 python/all-testaccounts.txt

Cargo.lock

@@ -163,9 +163,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.81"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
 dependencies = [
  "backtrace",
 ]
@@ -252,9 +252,9 @@ dependencies = [
 
 [[package]]
 name = "async-channel"
-version = "2.2.0"
+version = "2.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3"
+checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928"
 dependencies = [
  "concurrent-queue",
  "event-listener 5.2.0",
@@ -282,7 +282,7 @@ version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98892ebee4c05fc66757e600a7466f0d9bfcde338f645d64add323789f26cb36"
 dependencies = [
- "async-channel 2.2.0",
+ "async-channel 2.2.1",
  "base64 0.21.7",
  "bytes",
  "chrono",
@@ -344,7 +344,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -478,9 +478,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64"
-version = "0.22.0"
+version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
 [[package]]
 name = "base64ct"
@@ -724,9 +724,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.37"
+version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e"
+checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1059,7 +1059,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -1134,9 +1134,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
+checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
 
 [[package]]
 name = "default-net"
@@ -1162,13 +1162,13 @@ dependencies = [
  "ansi_term",
  "anyhow",
  "async-broadcast",
- "async-channel 2.2.0",
+ "async-channel 2.2.1",
  "async-imap",
  "async-native-tls",
  "async-smtp",
  "async_zip",
  "backtrace",
- "base64 0.22.0",
+ "base64 0.22.1",
  "brotli",
  "chrono",
  "criterion",
@@ -1239,9 +1239,10 @@ dependencies = [
 
 [[package]]
 name = "deltachat-contact-tools"
-version = "0.1.0"
+version = "0.0.0"
 dependencies = [
  "anyhow",
+ "chrono",
  "once_cell",
  "regex",
  "rusqlite",
@@ -1252,9 +1253,9 @@ name = "deltachat-jsonrpc"
 version = "1.137.4"
 dependencies = [
  "anyhow",
- "async-channel 2.2.0",
+ "async-channel 2.2.1",
  "axum",
- "base64 0.22.0",
+ "base64 0.22.1",
  "deltachat",
  "env_logger 0.11.3",
  "futures",
@@ -1312,7 +1313,7 @@ name = "deltachat_derive"
 version = "2.0.0"
 dependencies = [
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -1521,7 +1522,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -1830,7 +1831,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -1843,7 +1844,7 @@ dependencies = [
  "num-traits",
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -2164,7 +2165,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -2638,9 +2639,9 @@ dependencies = [
 
 [[package]]
 name = "imap-proto"
-version = "0.16.4"
+version = "0.16.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22e70cd66882c8cb1c9802096ba75212822153c51478dc61621e1a22f6c92361"
+checksum = "de555d9526462b6f9ece826a26fb7c67eca9a0245bd9ff84fa91972a5d5d8856"
 dependencies = [
  "nom",
 ]
@@ -2906,9 +2907,9 @@ dependencies = [
 
 [[package]]
 name = "mailparse"
-version = "0.14.1"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d096594926cab442e054e047eb8c1402f7d5b2272573b97ba68aa40629f9757"
+checksum = "3da03d5980411a724e8aaf7b61a7b5e386ec55a7fb49ee3d0ff79efc7e5e7c7e"
 dependencies = [
  "charset",
  "data-encoding",
@@ -3166,7 +3167,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -3227,7 +3228,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -3289,7 +3290,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -3530,7 +3531,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -3747,9 +3748,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.78"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
+checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba"
 dependencies = [
  "unicode-ident",
 ]
@@ -3860,9 +3861,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.35"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
  "proc-macro2",
 ]
@@ -4394,9 +4395,9 @@ dependencies = [
 
 [[package]]
 name = "schemars"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29"
+checksum = "fc6e7ed6919cb46507fb01ff1654309219f62b4d603822501b0b80d42f6f21ef"
 dependencies = [
  "dyn-clone",
  "schemars_derive",
@@ -4406,14 +4407,14 @@ dependencies = [
 
 [[package]]
 name = "schemars_derive"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967"
+checksum = "185f2b7aa7e02d418e453790dde16890256bbd2bcd04b7dc5348811052b53f49"
 dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 1.0.109",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -4500,9 +4501,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.197"
+version = "1.0.200"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
+checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f"
 dependencies = [
  "serde_derive",
 ]
@@ -4527,24 +4528,24 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.197"
+version = "1.0.200"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
+checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
 name = "serde_derive_internals"
-version = "0.26.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"
+checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -4818,7 +4819,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -4840,9 +4841,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.57"
+version = "2.0.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11a6ae1e52eb25aab8f3fb9fca13be982a373b8f1157ca14b897a825ba4a2d35"
+checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4962,22 +4963,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.58"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
+checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.58"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
+checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -5094,7 +5095,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -5267,7 +5268,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -5561,7 +5562,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
  "wasm-bindgen-shared",
 ]
 
@@ -5595,7 +5596,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5991,7 +5992,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]
@@ -6011,7 +6012,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.57",
+ "syn 2.0.60",
 ]
 
 [[package]]

Cargo.toml

@@ -40,7 +40,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
 async-broadcast = "0.7.0"
-async-channel = "2.0.0"
+async-channel = "2.2.1"
 async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
@@ -48,7 +48,7 @@ async_zip = { version = "0.0.12", default-features = false, features = ["deflate
 backtrace = "0.3"
 base64 = "0.22"
 brotli = { version = "5", default-features=false, features = ["std"] }
-chrono = { version = "0.4.37", default-features=false, features = ["clock", "std"] }
+chrono = { workspace = true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
@@ -64,7 +64,7 @@ iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5.3"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-mailparse = "0.14"
+mailparse = "0.15"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
@@ -168,7 +168,8 @@ harness = false
 anyhow = "1"
 once_cell = "1.18.0"
 regex = "1.10"
-rusqlite = { version = "0.31" }
+rusqlite = "0.31"
+chrono = { version = "0.4.38", default-features=false, features = ["alloc", "clock", "std"] }
 
 [features]
 default = ["vendored"]

deltachat-contact-tools/Cargo.toml

@@ -1,10 +1,9 @@
 [package]
 name = "deltachat-contact-tools"
-version = "0.1.0"
+version = "0.0.0" # No semver-stable versioning
 edition = "2021"
-description = "Contact-related tools, like parsing vcards and sanitizing name and address"
+description = "Contact-related tools, like parsing vcards and sanitizing name and address. Meant for internal use in the deltachat crate."
 license = "MPL-2.0"
-# TODO maybe it should be called "deltachat-text-utils" or similar?
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -13,6 +12,7 @@ anyhow = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
+chrono = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -29,10 +29,181 @@ use std::fmt;
 use std::ops::Deref;
 
 use anyhow::bail;
+use anyhow::Context as _;
 use anyhow::Result;
+use chrono::{DateTime, NaiveDateTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
 
+// TODOs to clean up:
+// - Check if sanitizing is done correctly everywhere
+// - Apply lints everywhere (https://doc.rust-lang.org/cargo/reference/workspaces.html#the-lints-table)
+
+#[derive(Debug)]
+/// A Contact, as represented in a VCard.
+pub struct VcardContact {
+    /// The email address, vcard property `email`
+    pub addr: String,
+    /// The contact's display name, vcard property `fn`
+    pub display_name: String,
+    /// The contact's public PGP key in Base64, vcard property `key`
+    pub key: Option<String>,
+    /// The contact's profile image (=avatar) in Base64, vcard property `photo`
+    pub profile_image: Option<String>,
+    /// The timestamp when the vcard was created / last updated, vcard property `rev`
+    pub timestamp: Result<u64>,
+}
+
+/// Returns a vCard containing given contacts.
+///
+/// Calling [`parse_vcard()`] on the returned result is a reverse operation.
+pub fn make_vcard(contacts: &[VcardContact]) -> String {
+    fn format_timestamp(c: &VcardContact) -> Option<String> {
+        let timestamp = *c.timestamp.as_ref().ok()?;
+        let timestamp: i64 = timestamp.try_into().ok()?;
+        let datetime = DateTime::from_timestamp(timestamp, 0)?;
+        Some(datetime.format("%Y%m%dT%H%M%SZ").to_string())
+    }
+
+    let mut res = "".to_string();
+    for c in contacts {
+        let addr = &c.addr;
+        let display_name = match c.display_name.is_empty() {
+            false => &c.display_name,
+            true => &c.addr,
+        };
+        res += &format!(
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             EMAIL:{addr}\n\
+             FN:{display_name}\n"
+        );
+        if let Some(key) = &c.key {
+            res += &format!("KEY:data:application/pgp-keys;base64,{key}\n");
+        }
+        if let Some(profile_image) = &c.profile_image {
+            res += &format!("PHOTO:data:image/jpeg;base64,{profile_image}\n");
+        }
+        if let Some(timestamp) = format_timestamp(c) {
+            res += &format!("REV:{timestamp}\n");
+        }
+        res += "END:VCARD\n";
+    }
+    res
+}
+
+/// Parses `VcardContact`s from a given `&str`.
+pub fn parse_vcard(vcard: &str) -> Result<Vec<VcardContact>> {
+    fn remove_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
+        let start_of_s = s.get(..prefix.len())?;
+
+        if start_of_s.eq_ignore_ascii_case(prefix) {
+            s.get(prefix.len()..)
+        } else {
+            None
+        }
+    }
+    fn vcard_property<'a>(s: &'a str, property: &str) -> Option<&'a str> {
+        let remainder = remove_prefix(s, property)?;
+        // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
+        // then `remainder` is now `;TYPE=work:alice@example.com`
+
+        // TODO this doesn't handle the case where there are quotes around a colon
+        let (params, value) = remainder.split_once(':')?;
+        // In the example from above, `params` is now `;TYPE=work`
+        // and `value` is now `alice@example.com`
+
+        if params
+            .chars()
+            .next()
+            .filter(|c| !c.is_ascii_punctuation() || *c == '_')
+            .is_some()
+        {
+            // `s` started with `property`, but the next character after it was not punctuation,
+            // so this line's property is actually something else
+            return None;
+        }
+        Some(value)
+    }
+    fn parse_datetime(datetime: &str) -> Result<u64> {
+        // According to https://www.rfc-editor.org/rfc/rfc6350#section-4.3.5, the timestamp
+        // is in ISO.8601.2004 format. DateTime::parse_from_rfc3339() apparently parses
+        // ISO.8601, but fails to parse any of the examples given.
+        // So, instead just parse using a format string.
+
+        // Parses 19961022T140000Z, 19961022T140000-05, or 19961022T140000-0500.
+        let timestamp = match DateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S%#z") {
+            Ok(datetime) => datetime.timestamp(),
+            // Parses 19961022T140000.
+            Err(e) => match NaiveDateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S") {
+                Ok(datetime) => datetime
+                    .and_local_timezone(chrono::offset::Local)
+                    .single()
+                    .context("Could not apply local timezone to parsed date and time")?
+                    .timestamp(),
+                Err(_) => return Err(e.into()),
+            },
+        };
+        Ok(timestamp.try_into()?)
+    }
+
+    let mut lines = vcard.lines().peekable();
+    let mut contacts = Vec::new();
+
+    while lines.peek().is_some() {
+        // Skip to the start of the vcard:
+        for line in lines.by_ref() {
+            if line.eq_ignore_ascii_case("BEGIN:VCARD") {
+                break;
+            }
+        }
+
+        let mut display_name = None;
+        let mut addr = None;
+        let mut key = None;
+        let mut photo = None;
+        let mut datetime = None;
+
+        for line in lines.by_ref() {
+            if let Some(email) = vcard_property(line, "email") {
+                addr.get_or_insert(email);
+            } else if let Some(name) = vcard_property(line, "fn") {
+                display_name.get_or_insert(name);
+            } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
+                .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
+                .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
+            {
+                key.get_or_insert(k);
+            } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
+                .or_else(|| remove_prefix(line, "PHOTO;TYPE=JPEG;ENCODING=b:"))
+                .or_else(|| remove_prefix(line, "PHOTO;ENCODING=BASE64;TYPE=JPEG:"))
+                .or_else(|| remove_prefix(line, "PHOTO:data:image/jpeg;base64,"))
+            {
+                photo.get_or_insert(p);
+            } else if let Some(rev) = vcard_property(line, "rev") {
+                datetime.get_or_insert(rev);
+            } else if line.eq_ignore_ascii_case("END:VCARD") {
+                break;
+            }
+        }
+
+        let (display_name, addr) =
+            sanitize_name_and_addr(display_name.unwrap_or(""), addr.unwrap_or(""));
+
+        contacts.push(VcardContact {
+            display_name,
+            addr,
+            key: key.map(|s| s.to_string()),
+            profile_image: photo.map(|s| s.to_string()),
+            timestamp: datetime
+                .context("No timestamp in vcard")
+                .and_then(parse_datetime),
+        });
+    }
+
+    Ok(contacts)
+}
+
 /// Valid contact address.
 #[derive(Debug, Clone)]
 pub struct ContactAddress(String);
@@ -81,14 +252,10 @@ impl rusqlite::types::ToSql for ContactAddress {
 /// Make the name and address
 pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
-    if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+    let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.is_empty() {
-                strip_rtlo_characters(
-                    &captures
-                        .get(1)
-                        .map_or("".to_string(), |m| normalize_name(m.as_str())),
-                )
+                strip_rtlo_characters(captures.get(1).map_or("", |m| m.as_str()))
             } else {
                 strip_rtlo_characters(name)
             },
@@ -97,8 +264,21 @@ pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (strip_rtlo_characters(name), addr.to_string())
+        (
+            strip_rtlo_characters(&normalize_name(name)),
+            addr.to_string(),
+        )
+    };
+    let mut name = normalize_name(&name);
+
+    // If the 'display name' is just the address, remove it:
+    // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
+    // If the display name is empty, DC will just show the address when it needs a display name.
+    if name == addr {
+        name = "".to_string();
     }
+
+    (name, addr)
 }
 
 /// Normalize a name.
@@ -230,8 +410,105 @@ impl rusqlite::types::ToSql for EmailAddress {
 
 #[cfg(test)]
 mod tests {
+    use chrono::TimeZone;
+
     use super::*;
 
+    #[test]
+    fn test_vcard_thunderbird() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN:'Alice Mueller'
+EMAIL;PREF=1:alice.mueller@posteo.de
+UID:a8083264-ca47-4be7-98a8-8ec3db1447ca
+END:VCARD
+BEGIN:VCARD
+VERSION:4.0
+FN:'bobzzz@freenet.de'
+EMAIL;PREF=1:bobzzz@freenet.de
+UID:cac4fef4-6351-4854-bbe4-9b6df857eaed
+END:VCARD
+",
+        )
+        .unwrap();
+
+        assert_eq!(contacts[0].addr, "alice.mueller@posteo.de".to_string());
+        assert_eq!(contacts[0].display_name, "Alice Mueller".to_string());
+        assert_eq!(contacts[0].key, None);
+        assert_eq!(contacts[0].profile_image, None);
+        assert!(contacts[0].timestamp.is_err());
+
+        assert_eq!(contacts[1].addr, "bobzzz@freenet.de".to_string());
+        assert_eq!(contacts[1].display_name, "".to_string());
+        assert_eq!(contacts[1].key, None);
+        assert_eq!(contacts[1].profile_image, None);
+        assert!(contacts[1].timestamp.is_err());
+
+        assert_eq!(contacts.len(), 2);
+    }
+
+    #[test]
+    fn test_vcard_simple_example() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:4.0
+FN:Alice Wonderland
+N:Wonderland;Alice;;;Ms.
+GENDER:W
+EMAIL;TYPE=work:alice@example.com
+KEY;TYPE=PGP;ENCODING=b:[base64-data]
+REV:20240418T184242Z
+
+END:VCARD",
+        )
+        .unwrap();
+
+        assert_eq!(contacts[0].addr, "alice@example.com".to_string());
+        assert_eq!(contacts[0].display_name, "Alice Wonderland".to_string());
+        assert_eq!(contacts[0].key, Some("[base64-data]".to_string()));
+        assert_eq!(contacts[0].profile_image, None);
+        assert_eq!(*contacts[0].timestamp.as_ref().unwrap(), 1713465762);
+
+        assert_eq!(contacts.len(), 1);
+    }
+
+    #[test]
+    fn test_make_and_parse_vcard() {
+        let contacts = [
+            VcardContact {
+                addr: "alice@example.org".to_string(),
+                display_name: "Alice Wonderland".to_string(),
+                key: Some("[base64-data]".to_string()),
+                profile_image: Some("image in Base64".to_string()),
+                timestamp: Ok(1713465762),
+            },
+            VcardContact {
+                addr: "bob@example.com".to_string(),
+                display_name: "".to_string(),
+                key: None,
+                profile_image: None,
+                timestamp: Ok(0),
+            },
+        ];
+        for len in 0..=contacts.len() {
+            let contacts = &contacts[0..len];
+            let vcard = make_vcard(contacts);
+            let parsed = parse_vcard(&vcard).unwrap();
+            assert_eq!(parsed.len(), contacts.len());
+            for i in 0..parsed.len() {
+                assert_eq!(parsed[i].addr, contacts[i].addr);
+                assert_eq!(parsed[i].display_name, contacts[i].display_name);
+                assert_eq!(parsed[i].key, contacts[i].key);
+                assert_eq!(parsed[i].profile_image, contacts[i].profile_image);
+                assert_eq!(
+                    parsed[i].timestamp.as_ref().unwrap(),
+                    contacts[i].timestamp.as_ref().unwrap()
+                );
+            }
+        }
+    }
+
     #[test]
     fn test_contact_address() -> Result<()> {
         let alice_addr = "alice@example.org";
@@ -277,4 +554,62 @@ mod tests {
         assert!(EmailAddress::new("u@tt").is_ok());
         assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
     }
+
+    #[test]
+    fn test_vcard_android() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD
+VERSION:2.1
+N:;Bob;;;
+FN:Bob
+TEL;CELL:+1-234-567-890
+EMAIL;HOME:bob@example.org
+END:VCARD
+BEGIN:VCARD
+VERSION:2.1
+N:;Alice;;;
+FN:Alice
+EMAIL;HOME:alice@example.org
+END:VCARD
+",
+        )
+        .unwrap();
+
+        assert_eq!(contacts[0].addr, "bob@example.org".to_string());
+        assert_eq!(contacts[0].display_name, "Bob".to_string());
+        assert_eq!(contacts[0].key, None);
+        assert_eq!(contacts[0].profile_image, None);
+
+        assert_eq!(contacts[1].addr, "alice@example.org".to_string());
+        assert_eq!(contacts[1].display_name, "Alice".to_string());
+        assert_eq!(contacts[1].key, None);
+        assert_eq!(contacts[1].profile_image, None);
+
+        assert_eq!(contacts.len(), 2);
+    }
+
+    #[test]
+    fn test_vcard_local_datetime() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD\n\
+             VERSION:4.0\n\
+             FN:Alice Wonderland\n\
+             EMAIL;TYPE=work:alice@example.org\n\
+             REV:20240418T184242\n\
+             END:VCARD",
+        )
+        .unwrap();
+        assert_eq!(contacts.len(), 1);
+        assert_eq!(contacts[0].addr, "alice@example.org".to_string());
+        assert_eq!(contacts[0].display_name, "Alice Wonderland".to_string());
+        assert_eq!(
+            *contacts[0].timestamp.as_ref().unwrap(),
+            chrono::offset::Local
+                .with_ymd_and_hms(2024, 4, 18, 18, 42, 42)
+                .unwrap()
+                .timestamp()
+                .try_into()
+                .unwrap()
+        );
+    }
 }

deltachat-ffi/deltachat.h

@@ -4390,9 +4390,9 @@ int             dc_msg_has_deviating_timestamp(const dc_msg_t* msg);
 
 
 /**
- * Check if a message has a location bound to it.
- * These messages are also returned by dc_get_locations()
- * and the UI may decide to display a special icon beside such messages,
+ * Check if a message has a POI location bound to it.
+ * These locations are also returned by dc_get_locations()
+ * The UI may decide to display a special icon beside such messages.
  *
  * @memberof dc_msg_t
  * @param msg The message object.

deltachat-jsonrpc/Cargo.toml

@@ -16,11 +16,11 @@ required-features = ["webserver"]
 anyhow = "1"
 deltachat = { path = ".." }
 num-traits = "0.2"
-schemars = "0.8.13"
+schemars = "0.8.19"
 serde = { version = "1.0", features = ["derive"] }
 tempfile = "3.10.1"
 log = "0.4"
-async-channel = { version = "2.0.0" }
+async-channel = { version = "2.2.1" }
 futures = { version = "0.3.30" }
 serde_json = "1"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }

deltachat-jsonrpc/src/api.rs

@@ -1771,6 +1771,29 @@ impl CommandApi {
         Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
     }
 
+    /// Sets Webxdc file as integration.
+    /// `file` is the .xdc to use as Webxdc integration.
+    async fn set_webxdc_integration(&self, account_id: u32, file_path: String) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.set_webxdc_integration(&file_path).await
+    }
+
+    /// Returns Webxdc instance used for optional integrations.
+    /// UI can open the Webxdc as usual.
+    /// Returns `None` if there is no integration; the caller can add one using `set_webxdc_integration` then.
+    /// `integrate_for` is the chat to get the integration for.
+    async fn init_webxdc_integration(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<Option<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx
+            .init_webxdc_integration(chat_id.map(ChatId::new))
+            .await?
+            .map(|msg_id| msg_id.to_u32()))
+    }
+
     /// Makes an HTTP GET request and returns a response.
     ///
     /// `url` is the HTTP or HTTPS URL.

deltachat-jsonrpc/src/api/types/message.rs

@@ -35,6 +35,10 @@ pub struct MessageObject {
     parent_id: Option<u32>,
 
     text: String,
+
+    /// Check if a message has a POI location bound to it.
+    /// These locations are also returned by `get_locations` method.
+    /// The UI may decide to display a special icon beside such messages.
     has_location: bool,
     has_html: bool,
     view_type: MessageViewtype,

deltachat-jsonrpc/typescript/package.json

@@ -25,7 +25,8 @@
   "exports": {
     ".": {
       "import": "./dist/deltachat.js",
-      "require": "./dist/deltachat.cjs"
+      "require": "./dist/deltachat.cjs",
+      "types": "./dist/deltachat.d.ts"
     }
   },
   "license": "MPL-2.0",

deltachat-rpc-client/examples/echobot.py

@@ -4,6 +4,7 @@
 it will echo back any text send to it, it also will print to console all Delta Chat core events.
 Pass --help to the CLI to see available options.
 """
+
 from deltachat_rpc_client import events, run_bot_cli
 
 hooks = events.HookCollection()

deltachat-rpc-client/examples/echobot_advanced.py

@@ -3,6 +3,7 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
+
 import logging
 import sys
 from threading import Thread

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -2,6 +2,7 @@
 """
 Example echo bot without using hooks
 """
+
 import logging
 import sys
 

deltachat-rpc-client/tox.ini

@@ -22,9 +22,8 @@ skipsdist = True
 skip_install = True
 deps =
     ruff
-    black
 commands =
-    black --quiet --check --diff src/ examples/ tests/
+    ruff format --quiet --diff src/ examples/ tests/
     ruff check src/ examples/ tests/
 
 [pytest]

deltachat-rpc-server/npm-package/.gitignore

@@ -0,0 +1,2 @@
+platform_package
+*.tgz

deltachat-rpc-server/npm-package/.npmignore

@@ -0,0 +1,3 @@
+platform_package/*
+scripts/
+*.tgz

deltachat-rpc-server/npm-package/README.md

@@ -0,0 +1,77 @@
+## npm package for deltachat-rpc-server
+
+This is the successor of `deltachat-node`,
+it does not use NAPI bindings but instead uses stdio executables
+to let you talk to core over jsonrpc over stdio.
+This simplifies cross-compilation and even reduces binary size (no CFFI layer and no NAPI layer).
+
+## Usage
+
+> The **minimum** nodejs version for this package is `20.11`
+
+```
+npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client
+```
+
+```js
+import { startDeltaChat } from "@deltachat/stdio-rpc-server";
+import { C } from "@deltachat/jsonrpc-client";
+
+async function main() {
+    const dc = await startDeltaChat("deltachat-data");
+    console.log(await dc.rpc.getSystemInfo());
+}
+```
+
+For a more complete example refer to https://github.com/deltachat-bot/echo/pull/69/files (TODO change link when pr is merged).
+
+## How to use on an unsupported platform
+
+<!-- todo instructions, will uses an env var for pointing to `deltachat-rpc-server` binary -->
+
+<!-- todo copy parts from https://github.com/deltachat/deltachat-desktop/blob/7045c6f549e4b9d5caa0709d5bd314bbd9fd53db/docs/UPDATE_CORE.md -->
+
+## How does it work when you install it
+
+NPM automatically installs platform dependent optional dependencies when `os` and `cpu` fields are set correctly.
+
+references:
+
+- https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages, [webarchive version](https://web.archive.org/web/20240309234250/https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages)
+- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#cpu
+- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#os
+
+When you import this package it searches for the rpc server in the following locations and order:
+
+1. `DELTA_CHAT_RPC_SERVER` environment variable
+2. in PATH
+   - unless `DELTA_CHAT_SKIP_PATH=1` is specified
+   - searches in .cargo/bin directory first
+   - but there an additional version check is performed
+3. prebuilds in npm packages
+
+## How do you built this package in CI
+
+- To build platform packages, run the `build_platform_package.py` script:
+  ```
+  python3 build_platform_package.py <cargo-target>
+  # example
+  python3 build_platform_package.py x86_64-apple-darwin
+  ```
+- Then pass it as an artifact to the last CI action that publishes the main package.
+- upload all packages from `deltachat-rpc-server/npm-package/platform_package`.
+- then publish `deltachat-rpc-server/npm-package`,
+  - this will run `update_optional_dependencies_and_version.js` (in the `prepack` script),
+    which puts all platform packages into `optionalDependencies` and updates the `version` in `package.json`
+
+## How to build a version you can use localy on your host machine for development
+
+You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have seperate scripts for making it work for local installation.
+
+- If you just need your host platform run `python scripts/make_local_dev_version.py`
+- note: this clears the `platform_package` folder
+- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple plaftorms with `build_platform_package.py`
+
+## Thanks to nlnet
+
+The initial work on this package was funded by nlnet as part of the [Delta Tauri](https://nlnet.nl/project/DeltaTauri/) Project.

deltachat-rpc-server/npm-package/index.d.ts

@@ -0,0 +1,39 @@
+import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
+
+export interface SearchOptions {
+  /** whether to disable looking for deltachat-rpc-server inside of $PATH */
+  skipSearchInPath: boolean;
+
+  /** whether to disable the DELTA_CHAT_RPC_SERVER environment variable */
+  disableEnvPath: boolean;
+}
+
+/**
+ * 
+ * @returns absolute path to deltachat-rpc-server binary
+ * @throws when it is not found
+ */
+export function getRPCServerPath(
+  options?: Partial<SearchOptions>
+): Promise<string>;
+
+
+
+export type DeltaChatOverJsonRpcServer = StdioDeltaChat & {
+    shutdown: () => Promise<void>;
+    readonly pathToServerBinary: string;
+};
+
+
+/**
+ * 
+ * @param directory directory for accounts folder
+ * @param options 
+ */
+export function startDeltaChat(directory: string, options?: Partial<SearchOptions> ): Promise<DeltaChatOverJsonRpcServer>
+
+
+export namespace FnTypes {
+    export type getRPCServerPath = typeof getRPCServerPath
+    export type startDeltaChat = typeof startDeltaChat
+}

deltachat-rpc-server/npm-package/index.js

@@ -0,0 +1,143 @@
+//@ts-check
+import { execFile, spawn } from "node:child_process";
+import { stat, readdir } from "node:fs/promises";
+import os from "node:os";
+import { join, basename } from "node:path";
+import process from "node:process";
+import { promisify } from "node:util";
+import {
+  ENV_VAR_NAME,
+  PATH_EXECUTABLE_NAME,
+  SKIP_SEARCH_IN_PATH,
+} from "./src/const.js";
+import {
+  ENV_VAR_LOCATION_NOT_FOUND,
+  FAILED_TO_START_SERVER_EXECUTABLE,
+  NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR,
+  NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
+} from "./src/errors.js";
+
+// Because this is not compiled by typescript, esm needs this stuff (` with { type: "json" };`,
+// nodejs still complains about it being experimental, but deno also uses it, so treefit bets taht it will become standard)
+import package_json from "./package.json" with { type: "json" };
+import { createRequire } from "node:module";
+
+// exports
+// - [ ] a raw starter that has a stdin/out handle thingie like desktop uses
+// - [X] a function that already wraps the stdio handle from above into the deltachat jsonrpc bindings
+
+function findRPCServerInNodeModules() {
+  const arch = os.arch();
+  const operating_system = process.platform;
+  const package_name = `@deltachat/stdio-rpc-server-${operating_system}-${arch}`;
+  try {
+    const { resolve } = createRequire(import.meta.url);
+    return resolve(package_name);
+  } catch (error) {
+    console.debug("findRpcServerInNodeModules", error);
+    if (Object.keys(package_json.optionalDependencies).includes(package_name)) {
+      throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
+    } else {
+      throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());
+    }
+  }
+}
+
+/** @type {import("./index").FnTypes.getRPCServerPath} */
+export async function getRPCServerPath(
+  options = { skipSearchInPath: false, disableEnvPath: false }
+) {
+  // @TODO: improve confusing naming of these options
+  const { skipSearchInPath, disableEnvPath } = options;
+  // 1. check if it is set as env var
+  if (process.env[ENV_VAR_NAME] && !disableEnvPath) {
+    try {
+      if (!(await stat(process.env[ENV_VAR_NAME])).isFile()) {
+        throw new Error(
+          `expected ${ENV_VAR_NAME} to point to the deltachat-rpc-server executable`
+        );
+      }
+    } catch (error) {
+      throw new Error(ENV_VAR_LOCATION_NOT_FOUND());
+    }
+    return process.env[ENV_VAR_NAME];
+  }
+
+  // 2. check if it can be found in PATH
+  if (!process.env[SKIP_SEARCH_IN_PATH] && !skipSearchInPath) {
+    const exec = promisify(execFile);
+
+    const { stdout: executable } =
+      os.platform() !== "win32"
+        ? await exec("command", ["-v", PATH_EXECUTABLE_NAME])
+        : await exec("where", [PATH_EXECUTABLE_NAME]);
+
+    // by just trying to execute it and then use "command -v deltachat-rpc-server" (unix) or "where deltachat-rpc-server" (windows) to get the path to the executable
+    if (executable.length > 1) {
+      // test if it is the right version
+      try {
+        // for some unknown reason it is in stderr and not in stdout
+        const { stderr } = await promisify(execFile)(executable, ["--version"]);
+        const version = stderr.slice(0, stderr.indexOf("\n"));
+        if (package_json.version !== version) {
+          throw new Error(
+            `version mismatch: (npm package: ${package_json.version})  (installed ${PATH_EXECUTABLE_NAME} version: ${version})`
+          );
+        } else {
+          return executable;
+        }
+      } catch (error) {
+        console.error(
+          "Found executable in PATH, but there was an error: " + error
+        );
+        console.error("So falling back to using prebuild...");
+      }
+    }
+  }
+  // 3. check for prebuilds
+
+  return findRPCServerInNodeModules();
+}
+
+import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
+
+/** @type {import("./index").FnTypes.startDeltaChat} */
+export async function startDeltaChat(directory, options) {
+  const pathToServerBinary = await getRPCServerPath(options);
+  const server = spawn(pathToServerBinary, {
+    env: {
+      RUST_LOG: process.env.RUST_LOG || "info",
+      DC_ACCOUNTS_PATH: directory,
+    },
+  });
+
+  server.on("error", (err) => {
+    throw new Error(FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, err));
+  });
+  let shouldClose = false;
+
+  server.on("exit", () => {
+    if (shouldClose) {
+      return;
+    }
+    throw new Error("Server quit");
+  });
+
+  server.stderr.pipe(process.stderr);
+
+  /** @type {import('./index').DeltaChatOverJsonRpcServer} */
+  //@ts-expect-error
+  const dc = new StdioDeltaChat(server.stdin, server.stdout, true);
+
+  dc.shutdown = async () => {
+    shouldClose = true;
+    if (!server.kill()) {
+      console.log("server termination failed");
+    }
+  };
+
+  //@ts-expect-error
+  dc.pathToServerBinary = pathToServerBinary;
+
+  return dc;
+}

deltachat-rpc-server/npm-package/package.json

@@ -0,0 +1,16 @@
+{
+    "type": "module",
+    "name": "@deltachat/stdio-rpc-server",
+    "version": "1.137.4",
+    "license": "MPL-2.0",
+    "main": "index.js",
+    "types": "index.d.ts",
+    "scripts": {
+        "prepack": "node scripts/update_optional_dependencies_and_version.js"
+    },
+    "optionalDependencies": {
+    },
+    "peerDependencies": {
+        "@deltachat/jsonrpc-client": "*"
+    }
+}

deltachat-rpc-server/npm-package/scripts/build_platform_package.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir
+from shutil import copy
+from src.make_package import write_package_json
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+if len(argv) < 2:
+    print("First argument should be target architecture as required by cargo")
+    exit(1)
+
+target = argv[1].strip()
+
+subprocess.run(
+    ["cargo", "build", "--release", "-p", "deltachat-rpc-server", "--target", target],
+    check=True,
+)
+
+newpath = "platform_package"
+if not path.exists(newpath):
+    makedirs(newpath)
+
+# make new folder
+
+platform_path = "platform_package/" + target
+if not path.exists(platform_path):
+    makedirs(platform_path)
+
+# copy binary it over
+
+
+def binary_path(binary_name):
+    return "../../target/" + target + "/release/" + binary_name
+
+
+my_binary_name = "deltachat-rpc-server"
+
+if not path.isfile(binary_path("deltachat-rpc-server")):
+    my_binary_name = "deltachat-rpc-server.exe"
+    if not path.isfile(binary_path("deltachat-rpc-server.exe")):
+        print("Did not find the build")
+        exit(1)
+
+my_binary_path = binary_path(my_binary_name)
+
+copy(my_binary_path, platform_path + "/" + my_binary_name)
+
+# make a package.json for it
+
+write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/make_local_dev_version.py

@@ -0,0 +1,31 @@
+# This script is for making a version of the npm packet that you can install locally
+
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir
+import re
+import json
+import tomllib
+from shutil import copy, rmtree
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+# get host target with "rustc -vV"
+output = subprocess.run(["rustc", "-vV"], capture_output=True)
+host_target = re.search('host: ([-\\w]*)', output.stdout.decode("utf-8")).group(1)
+print("host target to build for is:", host_target)
+
+# clean platform_package folder
+newpath = r'platform_package' 
+if not path.exists(newpath):
+    makedirs(newpath)
+else:
+    rmtree(path.join(path.dirname(path.abspath(__file__)), "../platform_package/"))
+    makedirs(newpath)
+
+# run build_platform_package.py with the host's target to build it
+subprocess.run(["python", "scripts/build_platform_package.py", host_target], capture_output=False, check=True)
+
+# run update_optional_dependencies_and_version.js to adjust the package / make it installable locally
+subprocess.run(["node", "scripts/update_optional_dependencies_and_version.js", "--local"], capture_output=False, check=True)

deltachat-rpc-server/npm-package/scripts/pack_binary_for_platform.py

@@ -0,0 +1,46 @@
+import subprocess
+from sys import argv
+from os import path, makedirs, chdir, chmod, stat
+import json
+from shutil import copy
+from src.make_package import write_package_json
+
+# ensure correct working directory
+chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
+
+if len(argv) < 3:
+    print("First argument should be target architecture as required by cargo")
+    print("Second argument should be the location of th built binary (binary_path)")
+    exit(1)
+
+target = argv[1].strip()
+binary_path = argv[2].strip()
+
+output = subprocess.run(["rustc","--print","target-list"], capture_output=True, check=True)
+available_targets = output.stdout.decode("utf-8")
+
+if available_targets.find(target) == -1:
+    print("target", target, "is not known / not valid")
+    exit(1)
+
+
+newpath = r'platform_package' 
+if not path.exists(newpath):
+    makedirs(newpath)
+
+# make new folder
+
+platform_path = 'platform_package/' + target
+if not path.exists(platform_path):
+    makedirs(platform_path)
+
+# copy binary it over
+
+my_binary_name = path.basename(binary_path)
+new_binary_path = platform_path + "/" + my_binary_name
+copy(binary_path, new_binary_path)
+chmod(new_binary_path, 0o555) # everyone can read & execute, nobody can write
+
+# make a package.json for it
+
+write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/src/convert_platform.py

@@ -0,0 +1,21 @@
+def convert_cpu_arch_to_npm_cpu_arch(arch):
+    if arch == "x86_64":
+        return "x64"
+    if arch == "i686":
+        return "i32"
+    if arch == "aarch64":
+        return "arm64"
+    if arch == "armv7" or arch == "arm":
+        return "arm"
+    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.arch':", arch)
+    return arch
+
+def convert_os_to_npm_os(os):
+    if os == "windows":
+        return "win32"
+    if os == "darwin" or os == "linux":
+        return os
+    if os.startswith("android"):
+        return "android"
+    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.platform':", os)
+    return os

deltachat-rpc-server/npm-package/scripts/src/make_package.py

@@ -0,0 +1,27 @@
+import tomllib
+import json
+
+from .convert_platform import convert_cpu_arch_to_npm_cpu_arch, convert_os_to_npm_os
+
+def write_package_json(platform_path, rust_target, my_binary_name):
+    if len(rust_target.split("-")) == 3:
+        [cpu_arch, vendor, os] = rust_target.split("-")
+    else:
+        [cpu_arch, vendor, os, _env] = rust_target.split("-")
+
+    # read version
+    tomlfile = open("../../Cargo.toml", 'rb')
+    version = tomllib.load(tomlfile)['package']['version']
+
+    package_json = dict({
+        "name": "@deltachat/stdio-rpc-server-" + convert_os_to_npm_os(os) + "-" + convert_cpu_arch_to_npm_cpu_arch(cpu_arch),
+        "version": version,
+        "os": [convert_os_to_npm_os(os)],
+        "cpu": [convert_cpu_arch_to_npm_cpu_arch(cpu_arch)],
+        "main": my_binary_name,
+        "license": "MPL-2.0"
+    })
+
+    file = open(platform_path + "/package.json", 'w')
+    file.write(json.dumps(package_json, indent=4))
+

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -0,0 +1,57 @@
+import fs from "node:fs/promises";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const expected_cwd = join(dirname(fileURLToPath(import.meta.url)), "..");
+
+if (process.cwd() !== expected_cwd) {
+  console.error(
+    "CWD missmatch: this script needs to be run from " + expected_cwd,
+    { actual: process.cwd(), expected: expected_cwd }
+  );
+  process.exit(1);
+}
+
+// whether to use local paths instead of npm registry version number for the prebuilds in optionalDependencies
+// useful for local development
+const is_local = process.argv.includes("--local");
+
+const package_json = JSON.parse(await fs.readFile("./package.json", "utf8"));
+
+const cargo_toml = await fs.readFile("../Cargo.toml", "utf8");
+const version = cargo_toml
+  .split("\n")
+  .find((line) => line.includes("version"))
+  .split('"')[1];
+
+const platform_packages_dir = "./platform_package";
+
+const platform_package_names = await Promise.all(
+  (await fs.readdir(platform_packages_dir)).map(async (name) => {
+    const p = JSON.parse(
+      await fs.readFile(
+        join(platform_packages_dir, name, "package.json"),
+        "utf8"
+      )
+    );
+    if (p.version !== version) {
+      console.error(
+        name,
+        "has a different version than the version of the rpc server.",
+        { rpc_server: version, platform_package: p.version }
+      );
+      throw new Error("version missmatch");
+    }
+    return { folder_name: name, package_name: p.name };
+  })
+);
+
+package_json.version = version;
+package_json.optionalDependencies = {};
+for (const { folder_name, package_name } of platform_package_names) {
+  package_json.optionalDependencies[package_name] = is_local
+    ? `file:${expected_cwd}/platform_package/${folder_name}` // npm seems to work better with an absolute path here
+    : version;
+}
+
+await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deltachat-rpc-server/npm-package/src/const.js

@@ -0,0 +1,6 @@
+//@ts-check
+
+export const PATH_EXECUTABLE_NAME = 'deltachat-rpc-server'
+
+export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"
+export const SKIP_SEARCH_IN_PATH = "DELTA_CHAT_SKIP_PATH"

deltachat-rpc-server/npm-package/src/errors.js

@@ -0,0 +1,41 @@
+//@ts-check
+import { ENV_VAR_NAME } from "./const.js";
+
+const cargoInstallCommand =
+  "cargo install --git https://github.com/deltachat/deltachat-core-rust deltachat-rpc-server";
+
+export function NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name) {
+  return `deltachat-rpc-server not found:
+
+- Install it with "npm i ${package_name}"
+- or download/compile deltachat-rpc-server for your platform and
+ - either put it into your PATH (for example with "${cargoInstallCommand}")
+ - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
+}
+
+export function NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR() {
+  return `deltachat-rpc-server not found:
+
+Unfortunately no prebuild is available for your system, so you need to provide deltachat-rpc-server yourself.
+
+- Download or Compile deltachat-rpc-server for your platform and
+ - either put it into your PATH (for example with "${cargoInstallCommand}")
+ - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
+}
+
+export function ENV_VAR_LOCATION_NOT_FOUND(error) {
+  return `deltachat-rpc-server not found in ${ENV_VAR_NAME}:
+
+    Error: ${error}
+
+    Content of ${ENV_VAR_NAME}: "${process.env[ENV_VAR_NAME]}"`;
+}
+
+export function FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, error) {
+  return `Failed to start server executable at '${pathToServerBinary}',
+
+  Error: ${error}
+
+Make sure the deltachat-rpc-server binary exists at this location 
+and you can start it with \`${pathToServerBinary} --version\``;
+}

node/lib/binding.ts

@@ -1,13 +1,9 @@
 import { join } from 'path'
-import * as url from 'url'
 
 /**
  * bindings are not typed yet.
  * if the available function names are required they can be found inside of `../src/module.c`
  */
-import build from 'node-gyp-build'
-export const bindings: any = build(
-  join(url.fileURLToPath(new URL('.', import.meta.url)), '../')
-)
+export const bindings: any = require('node-gyp-build')(join(__dirname, '../'))
 
 export default bindings

node/lib/chat.ts

@@ -1,11 +1,11 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
+import binding from './binding'
 import rawDebug from 'debug'
 const debug = rawDebug('deltachat:node:chat')
-import { C } from './constants.js'
-import { integerToHexColor } from './util.js'
-import { ChatJSON } from './types.js'
+import { C } from './constants'
+import { integerToHexColor } from './util'
+import { ChatJSON } from './types'
 
 interface NativeChat {}
 /**

node/lib/chatlist.ts

@@ -1,10 +1,9 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import { Lot } from './lot.js'
-import { Chat } from './chat.js'
-import rawDebug from 'debug'
-const debug = rawDebug('deltachat:node:chatlist')
+import binding from './binding'
+import { Lot } from './lot'
+import { Chat } from './chat'
+const debug = require('debug')('deltachat:node:chatlist')
 
 interface NativeChatList {}
 /**

node/lib/contact.ts

@@ -1,10 +1,9 @@
-import { integerToHexColor } from './util.js'
+import { integerToHexColor } from './util'
 
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import rawDebug from 'debug'
-const debug = rawDebug('deltachat:node:contact')
+import binding from './binding'
+const debug = require('debug')('deltachat:node:contact')
 
 interface NativeContact {}
 /**

node/lib/context.ts

@@ -1,17 +1,17 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import { C, EventId2EventName } from './constants.js'
-import { Chat } from './chat.js'
-import { ChatList } from './chatlist.js'
-import { Contact } from './contact.js'
-import { Message } from './message.js'
-import { Lot } from './lot.js'
-import { Locations } from './locations.js'
+import binding from './binding'
+import { C, EventId2EventName } from './constants'
+import { Chat } from './chat'
+import { ChatList } from './chatlist'
+import { Contact } from './contact'
+import { Message } from './message'
+import { Lot } from './lot'
+import { Locations } from './locations'
 import rawDebug from 'debug'
-import { AccountManager } from './deltachat.js'
+import { AccountManager } from './deltachat'
 import { join } from 'path'
-import { EventEmitter } from 'events'
+import { EventEmitter } from 'stream'
 const debug = rawDebug('deltachat:node:index')
 
 const noop = function () {}

node/lib/deltachat.ts

@@ -1,13 +1,13 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import { EventId2EventName } from './constants.js'
+import binding from './binding'
+import { EventId2EventName } from './constants'
 import { EventEmitter } from 'events'
 import { existsSync } from 'fs'
 import rawDebug from 'debug'
 import { tmpdir } from 'os'
 import { join } from 'path'
-import { Context } from './context.js'
+import { Context } from './context'
 const debug = rawDebug('deltachat:node:index')
 
 const noop = function () {}

node/lib/index.ts

@@ -1,20 +1,20 @@
-import { AccountManager } from './deltachat.js'
+import { AccountManager } from './deltachat'
 
 export default AccountManager
 
-export { Context } from './context.js'
-export { Chat } from './chat.js'
-export { ChatList } from './chatlist.js'
-export { C } from './constants.js'
-export { Contact } from './contact.js'
+export { Context } from './context'
+export { Chat } from './chat'
+export { ChatList } from './chatlist'
+export { C } from './constants'
+export { Contact } from './contact'
 export { AccountManager as DeltaChat }
-export { Locations } from './locations.js'
-export { Lot } from './lot.js'
+export { Locations } from './locations'
+export { Lot } from './lot'
 export {
   Message,
   MessageState,
   MessageViewType,
   MessageDownloadState,
-} from './message.js'
+} from './message'
 
-export * from './types.js'
+export * from './types'

node/lib/locations.ts

@@ -1,8 +1,7 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import rawDebug from 'debug'
-const debug = rawDebug('deltachat:node:locations')
+const binding = require('../binding')
+const debug = require('debug')('deltachat:node:locations')
 
 interface NativeLocations {}
 /**

node/lib/lot.ts

@@ -1,8 +1,7 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import rawDebug from 'debug'
-const debug = rawDebug('deltachat:node:lot')
+const binding = require('../binding')
+const debug = require('debug')('deltachat:node:lot')
 
 interface NativeLot {}
 /**

node/lib/message.ts

@@ -1,12 +1,11 @@
 /* eslint-disable camelcase */
 
-import binding from './binding.js'
-import { C } from './constants.js'
-import { Lot } from './lot.js'
-import { Chat } from './chat.js'
-import { WebxdcInfo } from './context.js'
-import rawDebug from 'debug'
-const debug = rawDebug('deltachat:node:message')
+import binding from './binding'
+import { C } from './constants'
+import { Lot } from './lot'
+import { Chat } from './chat'
+import { WebxdcInfo } from './context'
+const debug = require('debug')('deltachat:node:message')
 
 export enum MessageDownloadState {
   Available = C.DC_DOWNLOAD_AVAILABLE,

node/lib/node-gyp-build.d.ts

@@ -1 +0,0 @@
-declare module 'node-gyp-build'

node/lib/types.ts

@@ -1,4 +1,4 @@
-import { C } from './constants.js'
+import { C } from './constants'
 
 export type ChatTypes =
   | C.DC_CHAT_TYPE_GROUP

node/scripts/common.js

@@ -1,4 +1,4 @@
-import { spawnSync } from 'child_process'
+const spawnSync = require('child_process').spawnSync
 
 const verbose = isVerbose()
 
@@ -23,4 +23,4 @@ function isVerbose () {
   return loglevel === 'verbose' || process.env.CI === 'true'
 }
 
-export { spawn, log, isVerbose, verbose }
+module.exports = { spawn, log, isVerbose, verbose }

node/scripts/generate-constants.js

@@ -1,16 +1,14 @@
 #!/usr/bin/env node
-import fs from 'fs'
-import path from 'path'
-import * as url from 'url'
+const fs = require('fs')
+const path = require('path')
 
 const data = []
-const header = path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../../deltachat-ffi/deltachat.h')
+const header = path.resolve(__dirname, '../../deltachat-ffi/deltachat.h')
 
 console.log('Generating constants...')
 
 const header_data = fs.readFileSync(header, 'UTF-8')
 const regex = /^#define\s+(\w+)\s+(\w+)/gm
-var match
 while (null != (match = regex.exec(header_data))) {
   const key = match[1]
   const value = parseInt(match[2])
@@ -19,6 +17,8 @@ while (null != (match = regex.exec(header_data))) {
   }
 }
 
+delete header_data
+
 const constants = data
   .filter(
     ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
@@ -49,17 +49,17 @@ const events = data
 
 // backwards compat
 fs.writeFileSync(
-  path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../constants.js'),
+  path.resolve(__dirname, '../constants.js'),
   `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
 )
 // backwards compat
 fs.writeFileSync(
-  path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../events.js'),
+  path.resolve(__dirname, '../events.js'),
   `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
 )
 
 fs.writeFileSync(
-  path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../lib/constants.js'),
+  path.resolve(__dirname, '../lib/constants.ts'),
   `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
 // Generated!\n\nexport const EventId2EventName: { [key: number]: string } = {\n${events},\n}\n`
 )

node/scripts/install.js

@@ -1,7 +1,6 @@
-import {execSync} from 'child_process'
-import {existsSync} from 'fs'
-import {join} from 'path'
-import * as url from 'url'
+const {execSync} = require('child_process')
+const {existsSync} = require('fs')
+const {join} = require('path')
 
 const run = (cmd) => {
   console.log('[i] running `' + cmd + '`')
@@ -17,7 +16,7 @@ if (process.env.USE_SYSTEM_LIBDELTACHAT === 'true') {
   run('npm run install:prebuilds')
 }
 
-if (!existsSync(join(url.fileURLToPath(new URL('.', import.meta.url)), '..', 'dist'))) {
+if (!existsSync(join(__dirname, '..', 'dist'))) {
   console.log('[i] Didn\'t find already built typescript bindings. Trying to transpile them. If this fail, make sure typescript is installed ;)')
   run('npm run build:bindings:ts')
 }

node/scripts/postLinksToDetails.js

@@ -1,5 +1,4 @@
-import { readFileSync } from 'fs'
-import { request } from 'https'
+const { readFileSync } = require('fs')
 
 const sha = JSON.parse(
   readFileSync(process.env['GITHUB_EVENT_PATH'], 'utf8')
@@ -22,6 +21,8 @@ const STATUS_DATA = {
   target_url: base_url + file_url,
 }
 
+const http = require('https')
+
 const options = {
   method: 'POST',
   headers: {
@@ -31,7 +32,7 @@ const options = {
   },
 }
 
-const req = request(GITHUB_API_URL, options, function (res) {
+const req = http.request(GITHUB_API_URL, options, function(res) {
   var chunks = []
   res.on('data', function(chunk) {
     chunks.push(chunk)

node/scripts/postinstall.js

@@ -1,6 +1,5 @@
-import fs from 'fs'
-import path from 'path'
-import * as url from 'url'
+const fs = require('fs')
+const path = require('path')
 
 if (process.platform !== 'win32') {
   console.log('postinstall: not windows, so skipping!')
@@ -8,7 +7,7 @@ if (process.platform !== 'win32') {
 }
 
 const from = path.resolve(
-  url.fileURLToPath(new URL('.', import.meta.url)),
+  __dirname,
   '..',
   '..',
   'target',
@@ -20,7 +19,7 @@ const getDestination = () => {
   const argv = process.argv
   if (argv.length === 3 && argv[2] === '--prebuild') {
     return path.resolve(
-      url.fileURLToPath(new URL('.', import.meta.url)),
+      __dirname,
       '..',
       'prebuilds',
       'win32-x64',
@@ -28,7 +27,7 @@ const getDestination = () => {
     )
   } else {
     return path.resolve(
-      url.fileURLToPath(new URL('.', import.meta.url)),
+      __dirname,
       '..',
       'build',
       'Release',

node/scripts/rebuild-core.js

@@ -1,8 +1,7 @@
-import path from 'path'
-import { spawn } from './common.js'
-import * as url from 'url'
+const path = require('path')
+const { spawn } = require('./common')
 const opts = {
-  cwd: path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../..'),
+  cwd: path.resolve(__dirname, '../..'),
   stdio: 'inherit'
 }
 

node/tsconfig.json

@@ -3,8 +3,7 @@
         "outDir": "dist",
         "rootDir": "./lib",
         "sourceMap": true,
-        "module": "nodenext",
-        "moduleResolution": "nodenext",
+        "module": "commonjs",
         "target": "es5",
         "esModuleInterop": true,
         "declaration": true,

package.json

@@ -1,5 +1,4 @@
 {
-  "type": "module",
   "dependencies": {
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
@@ -11,7 +10,7 @@
     "@types/node": "^20.8.10",
     "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
-    "mocha": "^10.2.0",
+    "mocha": "^8.2.1",
     "node-gyp": "^10.0.0",
     "prebuildify": "^5.0.1",
     "prebuildify-ci": "^1.0.5",
@@ -28,7 +27,7 @@
   ],
   "homepage": "https://github.com/deltachat/deltachat-core-rust/tree/master/node",
   "license": "GPL-3.0-or-later",
-  "exports": "./node/dist/index.js",
+  "main": "node/dist/index.js",
   "name": "deltachat-node",
   "repository": {
     "type": "git",

python/tests/test_1_online.py

@@ -2036,14 +2036,15 @@ def test_send_receive_locations(acfactory, lp):
     assert chat1.is_sending_locations()
     ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
+    # Wait for "enabled location streaming" message.
+    ac2._evtracker.wait_next_incoming_message()
+
+    # First location is sent immediately as a location-only message.
     ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
     ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
-    chat1.send_text("🍞")
     ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
     lp.sec("ac2: wait for incoming location message")
-
-    # currently core emits location changed before event_incoming message
     ac2._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
 
     locations = chat2.get_locations()
@@ -2052,7 +2053,7 @@ def test_send_receive_locations(acfactory, lp):
     assert locations[0].longitude == 3.0
     assert locations[0].accuracy == 0.5
     assert locations[0].timestamp > now
-    assert locations[0].marker == "🍞"
+    assert locations[0].marker is None
 
     contact = ac2.create_contact(ac1)
     locations2 = chat2.get_locations(contact=contact)

python/tox.ini

@@ -41,12 +41,11 @@ skipsdist = True
 skip_install = True
 deps =
     ruff
-    black
     # pygments required by rst-lint
     pygments
     restructuredtext_lint 
 commands =
-    black --quiet --check --diff setup.py src/deltachat examples/ tests/
+    ruff format --quiet --diff setup.py src/deltachat examples/ tests/
     ruff check src/deltachat tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.77.1
+RUST_VERSION=1.78.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/set_core_version.py

@@ -66,7 +66,11 @@ def main():
     parser = ArgumentParser(prog="set_core_version")
     parser.add_argument("newversion")
 
-    json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"]
+    json_list = [
+        "package.json",
+        "deltachat-jsonrpc/typescript/package.json",
+        "deltachat-rpc-server/npm-package/package.json",
+    ]
     toml_list = [
         "Cargo.toml",
         "deltachat-ffi/Cargo.toml",

src/authres.rs

@@ -14,7 +14,6 @@ use once_cell::sync::Lazy;
 use crate::config::Config;
 use crate::context::Context;
 use crate::headerdef::HeaderDef;
-use crate::tools::time;
 
 /// `authres` is short for the Authentication-Results header, defined in
 /// <https://datatracker.ietf.org/doc/html/rfc8601>, which contains info
@@ -29,45 +28,28 @@ pub(crate) async fn handle_authres(
     context: &Context,
     mail: &ParsedMail<'_>,
     from: &str,
-    message_time: i64,
 ) -> Result<DkimResults> {
     let from_domain = match EmailAddress::new(from) {
         Ok(email) => email.domain,
         Err(e) => {
-            // This email is invalid, but don't return an error, we still want to
-            // add a stub to the database so that it's not downloaded again
             return Err(anyhow::format_err!("invalid email {}: {:#}", from, e));
         }
     };
 
     let authres = parse_authres_headers(&mail.get_headers(), &from_domain);
     update_authservid_candidates(context, &authres).await?;
-    compute_dkim_results(context, authres, &from_domain, message_time).await
+    compute_dkim_results(context, authres).await
 }
 
 #[derive(Debug)]
 pub(crate) struct DkimResults {
     /// Whether DKIM passed for this particular e-mail.
     pub dkim_passed: bool,
-    /// Whether DKIM is known to work for e-mails coming from the sender's domain,
-    /// i.e. whether we expect DKIM to work.
-    pub dkim_should_work: bool,
-    /// Whether changing the public Autocrypt key should be allowed.
-    /// This is false if we expected DKIM to work (dkim_works=true),
-    /// but it failed now (dkim_passed=false).
-    pub allow_keychange: bool,
 }
 
 impl fmt::Display for DkimResults {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            fmt,
-            "DKIM Results: Passed={}, Works={}, Allow_Keychange={}",
-            self.dkim_passed, self.dkim_should_work, self.allow_keychange
-        )?;
-        if !self.allow_keychange {
-            write!(fmt, " KEYCHANGES NOT ALLOWED!!!!")?;
-        }
+        write!(fmt, "DKIM Results: Passed={}", self.dkim_passed)?;
         Ok(())
     }
 }
@@ -218,10 +200,6 @@ async fn update_authservid_candidates(
         context
             .set_config_internal(Config::AuthservIdCandidates, Some(&new_config))
             .await?;
-        // Updating the authservid candidates may mean that we now consider
-        // emails as "failed" which "passed" previously, so we need to
-        // reset our expectation which DKIMs work.
-        clear_dkim_works(context).await?
     }
     Ok(())
 }
@@ -238,8 +216,6 @@ async fn update_authservid_candidates(
 async fn compute_dkim_results(
     context: &Context,
     mut authres: ParsedAuthresHeaders,
-    from_domain: &str,
-    message_time: i64,
 ) -> Result<DkimResults> {
     let mut dkim_passed = false;
 
@@ -272,71 +248,7 @@ async fn compute_dkim_results(
         }
     }
 
-    let last_working_timestamp = dkim_works_timestamp(context, from_domain).await?;
-    let mut dkim_should_work = dkim_should_work(last_working_timestamp)?;
-    if message_time > last_working_timestamp && dkim_passed {
-        set_dkim_works_timestamp(context, from_domain, message_time).await?;
-        dkim_should_work = true;
-    }
-
-    Ok(DkimResults {
-        dkim_passed,
-        dkim_should_work,
-        allow_keychange: dkim_passed || !dkim_should_work,
-    })
-}
-
-/// Whether DKIM in emails from this domain should be considered to work.
-fn dkim_should_work(last_working_timestamp: i64) -> Result<bool> {
-    // When we get an email with valid DKIM-Authentication-Results,
-    // then we assume that DKIM works for 30 days from this time on.
-    let should_work_until = last_working_timestamp + 3600 * 24 * 30;
-
-    let dkim_ever_worked = last_working_timestamp > 0;
-
-    // We're using time() here and not the time when the message
-    // claims to have been sent (passed around as `message_time`)
-    // because otherwise an attacker could just put a time way
-    // in the future into the `Date` header and then we would
-    // assume that DKIM doesn't have to be valid anymore.
-    let dkim_should_work_now = should_work_until > time();
-    Ok(dkim_ever_worked && dkim_should_work_now)
-}
-
-async fn dkim_works_timestamp(context: &Context, from_domain: &str) -> Result<i64, anyhow::Error> {
-    let last_working_timestamp: i64 = context
-        .sql
-        .query_get_value(
-            "SELECT dkim_works FROM sending_domains WHERE domain=?",
-            (from_domain,),
-        )
-        .await?
-        .unwrap_or(0);
-    Ok(last_working_timestamp)
-}
-
-async fn set_dkim_works_timestamp(
-    context: &Context,
-    from_domain: &str,
-    timestamp: i64,
-) -> Result<()> {
-    context
-        .sql
-        .execute(
-            "INSERT INTO sending_domains (domain, dkim_works) VALUES (?,?)
-                ON CONFLICT(domain) DO UPDATE SET dkim_works=excluded.dkim_works",
-            (from_domain, timestamp),
-        )
-        .await?;
-    Ok(())
-}
-
-async fn clear_dkim_works(context: &Context) -> Result<()> {
-    context
-        .sql
-        .execute("DELETE FROM sending_domains", ())
-        .await?;
-    Ok(())
+    Ok(DkimResults { dkim_passed })
 }
 
 fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str> {
@@ -349,19 +261,12 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
-    use std::time::Duration;
-
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 
     use super::*;
-    use crate::aheader::EncryptPreference;
-    use crate::e2ee;
     use crate::mimeparser;
     use crate::peerstate::Peerstate;
-    use crate::securejoin::get_securejoin_qr;
-    use crate::securejoin::join_securejoin;
-    use crate::test_utils;
     use crate::test_utils::TestContext;
     use crate::test_utils::TestContextManager;
     use crate::tools;
@@ -574,33 +479,8 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 let mail = mailparse::parse_mail(&bytes)?;
                 let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
 
-                let res = handle_authres(&t, &mail, from, time()).await?;
-                assert!(res.allow_keychange);
-            }
-
-            for entry in &dir {
-                let mut file = fs::File::open(entry.path()).await?;
-                bytes.clear();
-                file.read_to_end(&mut bytes).await.unwrap();
-
-                let mail = mailparse::parse_mail(&bytes)?;
-                let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
-
-                let res = handle_authres(&t, &mail, from, time()).await?;
-                if !res.allow_keychange {
-                    println!(
-                        "!!!!!! FAILURE Receiving {:?}, keychange is not allowed !!!!!!",
-                        entry.path()
-                    );
-                    test_failed = true;
-                }
-
+                let res = handle_authres(&t, &mail, from).await?;
                 let from_domain = EmailAddress::new(from).unwrap().domain;
-                assert_eq!(
-                    res.dkim_should_work,
-                    dkim_should_work(dkim_works_timestamp(&t, &from_domain).await?)?
-                );
-                assert_eq!(res.dkim_passed, res.dkim_should_work);
 
                 // delta.blinzeln.de and gmx.de have invalid DKIM, so the DKIM check should fail
                 let expected_result = (from_domain != "delta.blinzeln.de") && (from_domain != "gmx.de")
@@ -613,9 +493,8 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 if res.dkim_passed != expected_result {
                     if authres_parsing_works {
                         println!(
-                            "!!!!!! FAILURE Receiving {:?}, order {:#?} wrong result: !!!!!!",
+                            "!!!!!! FAILURE Receiving {:?} wrong result: !!!!!!",
                             entry.path(),
-                            dir.iter().map(|e| e.file_name()).collect::<Vec<_>>()
                         );
                         test_failed = true;
                     }
@@ -638,116 +517,7 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
         let bytes = b"From: invalid@from.com
 Authentication-Results: dkim=";
         let mail = mailparse::parse_mail(bytes).unwrap();
-        handle_authres(&t, &mail, "invalid@rom.com", time())
-            .await
-            .unwrap();
-    }
-
-    #[ignore = "Disallowing keychanges is disabled for now"]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_handle_authres_fails() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = tcm.alice().await;
-        let bob = tcm.bob().await;
-
-        // Bob sends Alice a message, so she gets his key
-        tcm.send_recv_accept(&bob, &alice, "Hi").await;
-
-        // We don't need bob anymore, let's make sure it's not accidentally used
-        drop(bob);
-
-        // Assume Alice receives an email from bob@example.net with
-        // correct DKIM -> `set_dkim_works()` was called
-        set_dkim_works_timestamp(&alice, "example.net", time()).await?;
-        // And Alice knows her server's authserv-id
-        alice
-            .set_config(Config::AuthservIdCandidates, Some("example.org"))
-            .await?;
-
-        tcm.section("An attacker, bob2, sends a from-forged email to Alice!");
-
-        // Sleep to make sure key reset is ignored because of DKIM failure
-        // and not because reordering is suspected.
-        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
-
-        let bob2 = tcm.unconfigured().await;
-        bob2.configure_addr("bob@example.net").await;
-        e2ee::ensure_secret_key_exists(&bob2).await?;
-
-        let chat = bob2.create_chat(&alice).await;
-        let mut sent = bob2
-            .send_text(chat.id, "Please send me lots of money")
-            .await;
-
-        sent.payload
-            .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
-
-        let received = alice.recv_msg(&sent).await;
-
-        // Assert that the error tells the user about the problem
-        assert!(received.error.unwrap().contains("DKIM failed"));
-
-        let bob_state = Peerstate::from_addr(&alice, "bob@example.net")
-            .await?
-            .unwrap();
-
-        // Encryption preference is still mutual.
-        assert_eq!(bob_state.prefer_encrypt, EncryptPreference::Mutual);
-
-        // Also check that the keypair was not changed
-        assert_eq!(
-            bob_state.public_key.unwrap(),
-            test_utils::bob_keypair().public
-        );
-
-        // Since Alice didn't change the key, Bob can't read her message
-        let received = tcm
-            .try_send_recv(&alice, &bob2, "My credit card number is 1234")
-            .await;
-        assert!(!received.text.contains("1234"));
-        assert!(received.error.is_some());
-
-        tcm.section("Turns out bob2 wasn't an attacker at all, Bob just has a new phone and DKIM just stopped working.");
-        tcm.section("To fix the key problems, Bob scans Alice's QR code.");
-
-        let qr = get_securejoin_qr(&alice.ctx, None).await.unwrap();
-        join_securejoin(&bob2.ctx, &qr).await.unwrap();
-
-        loop {
-            if let Some(mut sent) = bob2.pop_sent_msg_opt(Duration::ZERO).await {
-                sent.payload
-                    .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
-                alice.recv_msg(&sent).await;
-            } else if let Some(sent) = alice.pop_sent_msg_opt(Duration::ZERO).await {
-                bob2.recv_msg(&sent).await;
-            } else {
-                break;
-            }
-        }
-
-        // Unfortunately, securejoin currently doesn't work with authres-checking,
-        // so these checks would fail:
-
-        // let contact_bob = alice.add_or_lookup_contact(&bob2).await;
-        // assert_eq!(
-        //     contact_bob.is_verified(&alice.ctx).await.unwrap(),
-        //     VerifiedStatus::BidirectVerified
-        // );
-
-        // let contact_alice = bob2.add_or_lookup_contact(&alice).await;
-        // assert_eq!(
-        //     contact_alice.is_verified(&bob2.ctx).await.unwrap(),
-        //     VerifiedStatus::BidirectVerified
-        // );
-
-        // // Bob can read Alice's messages again
-        // let received = tcm
-        //     .try_send_recv(&alice, &bob2, "Can you read this again?")
-        //     .await;
-        // assert_eq!(received.text.as_ref().unwrap(), "Can you read this again?");
-        // assert!(received.error.is_none());
-
-        Ok(())
+        handle_authres(&t, &mail, "invalid@rom.com").await.unwrap();
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -796,10 +566,7 @@ Authentication-Results: dkim=";
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
 
-        // Assume Bob received an email from something@example.net with
-        // correct DKIM -> `set_dkim_works()` was called
-        set_dkim_works_timestamp(&bob, "example.org", time()).await?;
-        // And Bob knows his server's authserv-id
+        // Bob knows his server's authserv-id
         bob.set_config(Config::AuthservIdCandidates, Some("example.net"))
             .await?;
 
@@ -821,15 +588,13 @@ Authentication-Results: dkim=";
             .insert_str(0, "Authentication-Results: example.net; dkim=fail\n");
         let rcvd = bob.recv_msg(&sent).await;
 
-        // Disallowing keychanges is disabled for now:
-        // assert!(rcvd.error.unwrap().contains("DKIM failed"));
         // The message info should contain a warning:
         assert!(rcvd
             .id
             .get_info(&bob)
             .await
             .unwrap()
-            .contains("KEYCHANGES NOT ALLOWED"));
+            .contains("DKIM Results: Passed=false"));
 
         Ok(())
     }

src/blob.rs

@@ -1330,6 +1330,42 @@ mod tests {
         Ok(img)
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_send_big_gif_as_image() -> Result<()> {
+        let bytes = include_bytes!("../test-data/image/screenshot.gif");
+        let (width, height) = (1920u32, 1080u32);
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        alice
+            .set_config(
+                Config::MediaQuality,
+                Some(&(MediaQuality::Worse as i32).to_string()),
+            )
+            .await?;
+        let file = alice.get_blobdir().join("file").with_extension("gif");
+        fs::write(&file, &bytes)
+            .await
+            .context("failed to write file")?;
+        let mut msg = Message::new(Viewtype::Image);
+        msg.set_file(file.to_str().unwrap(), None);
+        let chat = alice.create_chat(&bob).await;
+        let sent = alice.send_msg(chat.id, &mut msg).await;
+        let bob_msg = bob.recv_msg(&sent).await;
+        // DC must detect the image as GIF and send it w/o reencoding.
+        assert_eq!(bob_msg.get_viewtype(), Viewtype::Gif);
+        assert_eq!(bob_msg.get_width() as u32, width);
+        assert_eq!(bob_msg.get_height() as u32, height);
+        let file_saved = bob
+            .get_blobdir()
+            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+        bob_msg.save_file(&bob, &file_saved).await?;
+        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
+        let (file_size, _) = blob.metadata()?;
+        assert_eq!(file_size, bytes.len() as u64);
+        check_image_size(file_saved, width, height);
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_in_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;

src/chat.rs

@@ -2235,8 +2235,9 @@ pub struct ChatInfo {
 }
 
 pub(crate) async fn update_saved_messages_icon(context: &Context) -> Result<()> {
-    // if there is no saved-messages chat, there is nothing to update. this is no error.
-    if let Some(chat_id) = ChatId::lookup_by_contact(context, ContactId::SELF).await? {
+    if let Some(ChatIdBlocked { id: chat_id, .. }) =
+        ChatIdBlocked::lookup_by_contact(context, ContactId::SELF).await?
+    {
         let icon = include_bytes!("../assets/icon-saved-messages.png");
         let blob = BlobObject::create(context, "icon-saved-messages.png", icon).await?;
         let icon = blob.as_name().to_string();
@@ -2249,8 +2250,9 @@ pub(crate) async fn update_saved_messages_icon(context: &Context) -> Result<()>
 }
 
 pub(crate) async fn update_device_icon(context: &Context) -> Result<()> {
-    // if there is no device-chat, there is nothing to update. this is no error.
-    if let Some(chat_id) = ChatId::lookup_by_contact(context, ContactId::DEVICE).await? {
+    if let Some(ChatIdBlocked { id: chat_id, .. }) =
+        ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE).await?
+    {
         let icon = include_bytes!("../assets/icon-device.png");
         let blob = BlobObject::create(context, "icon-device.png", icon).await?;
         let icon = blob.as_name().to_string();
@@ -2301,7 +2303,9 @@ async fn update_special_chat_name(
     contact_id: ContactId,
     name: String,
 ) -> Result<()> {
-    if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
+    if let Some(ChatIdBlocked { id: chat_id, .. }) =
+        ChatIdBlocked::lookup_by_contact(context, contact_id).await?
+    {
         // the `!= name` condition avoids unneeded writes
         context
             .sql
@@ -2505,6 +2509,30 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
             .await?
             .with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?;
 
+        if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
+            // Correct the type, take care not to correct already very special
+            // formats as GIF or VOICE.
+            //
+            // Typical conversions:
+            // - from FILE to AUDIO/VIDEO/IMAGE
+            // - from FILE/IMAGE to GIF */
+            if let Some((better_type, _)) = message::guess_msgtype_from_suffix(&blob.to_abs_path())
+            {
+                if better_type != Viewtype::Webxdc
+                    || context
+                        .ensure_sendable_webxdc_file(&blob.to_abs_path())
+                        .await
+                        .is_ok()
+                {
+                    msg.viewtype = better_type;
+                }
+            }
+        } else if msg.viewtype == Viewtype::Webxdc {
+            context
+                .ensure_sendable_webxdc_file(&blob.to_abs_path())
+                .await?;
+        }
+
         let mut maybe_sticker = msg.viewtype == Viewtype::Sticker;
         if msg.viewtype == Viewtype::Image
             || maybe_sticker && !msg.param.exists(Param::ForceSticker)
@@ -2526,34 +2554,6 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
                 .set(Param::Filename, stem.to_string() + "." + blob_ext);
         }
 
-        if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
-            // Correct the type, take care not to correct already very special
-            // formats as GIF or VOICE.
-            //
-            // Typical conversions:
-            // - from FILE to AUDIO/VIDEO/IMAGE
-            // - from FILE/IMAGE to GIF */
-            if let Some((better_type, better_mime)) =
-                message::guess_msgtype_from_suffix(&blob.to_abs_path())
-            {
-                if better_type != Viewtype::Webxdc
-                    || context
-                        .ensure_sendable_webxdc_file(&blob.to_abs_path())
-                        .await
-                        .is_ok()
-                {
-                    msg.viewtype = better_type;
-                    if !msg.param.exists(Param::MimeType) {
-                        msg.param.set(Param::MimeType, better_mime);
-                    }
-                }
-            }
-        } else if msg.viewtype == Viewtype::Webxdc {
-            context
-                .ensure_sendable_webxdc_file(&blob.to_abs_path())
-                .await?;
-        }
-
         if !msg.param.exists(Param::MimeType) {
             if let Some((_, mime)) = message::guess_msgtype_from_suffix(&blob.to_abs_path()) {
                 msg.param.set(Param::MimeType, mime);
@@ -2597,6 +2597,18 @@ async fn prepare_msg_common(
         }
     }
 
+    // Check a quote reply is not leaking data from other chats.
+    // This is meant as a last line of defence, the UI should check that before as well.
+    // (We allow Chattype::Single in general for "Reply Privately";
+    // checking for exact contact_id will produce false positives when ppl just left the group)
+    if chat.typ != Chattype::Single && !context.get_config_bool(Config::Bot).await? {
+        if let Some(quoted_message) = msg.quoted_message(context).await? {
+            if quoted_message.chat_id != chat_id {
+                bail!("Bad quote reply");
+            }
+        }
+    }
+
     // check current MessageState for drafts (to keep msg_id) ...
     let update_msg_id = if msg.state == MessageState::OutDraft {
         msg.hidden = false;
@@ -2839,17 +2851,10 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             .await?;
     }
 
-    if let Some(last_added_location_id) = rendered_msg.last_added_location_id {
+    if rendered_msg.last_added_location_id.is_some() {
         if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, now).await {
             error!(context, "Failed to set kml sent_timestamp: {err:#}.");
         }
-        if !msg.hidden {
-            if let Err(err) =
-                location::set_msg_location_id(context, msg.id, last_added_location_id).await
-            {
-                error!(context, "Failed to set msg_location_id: {err:#}.");
-            }
-        }
     }
 
     if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
@@ -4475,9 +4480,10 @@ impl Context {
                     }
                     _ => (),
                 }
-                ChatId::lookup_by_contact(self, contact_id)
+                ChatIdBlocked::lookup_by_contact(self, contact_id)
                     .await?
                     .with_context(|| format!("No chat for addr '{addr}'"))?
+                    .id
             }
             SyncId::Grpid(grpid) => {
                 if let SyncAction::CreateBroadcast(name) = action {
@@ -4724,6 +4730,59 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_quote_replies() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let grp_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?;
+        let grp_msg_id = send_text_msg(&alice, grp_chat_id, "bar".to_string()).await?;
+        let grp_msg = Message::load_from_db(&alice, grp_msg_id).await?;
+
+        let one2one_chat_id = alice.create_chat(&bob).await.id;
+        let one2one_msg_id = send_text_msg(&alice, one2one_chat_id, "foo".to_string()).await?;
+        let one2one_msg = Message::load_from_db(&alice, one2one_msg_id).await?;
+
+        // quoting messages in same chat is okay
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("baz".to_string());
+        msg.set_quote(&alice, Some(&grp_msg)).await?;
+        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
+        assert!(result.is_ok());
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("baz".to_string());
+        msg.set_quote(&alice, Some(&one2one_msg)).await?;
+        let result = send_msg(&alice, one2one_chat_id, &mut msg).await;
+        assert!(result.is_ok());
+        let one2one_quote_reply_msg_id = result.unwrap();
+
+        // quoting messages from groups to one-to-ones is okay ("reply privately")
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("baz".to_string());
+        msg.set_quote(&alice, Some(&grp_msg)).await?;
+        let result = send_msg(&alice, one2one_chat_id, &mut msg).await;
+        assert!(result.is_ok());
+
+        // quoting messages from one-to-one chats in groups is an error; usually this is also not allowed by UI at all ...
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text("baz".to_string());
+        msg.set_quote(&alice, Some(&one2one_msg)).await?;
+        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
+        assert!(result.is_err());
+
+        // ... but forwarding messages with quotes is allowed
+        let result = forward_msgs(&alice, &[one2one_quote_reply_msg_id], grp_chat_id).await;
+        assert!(result.is_ok());
+
+        // ... and bots are not restricted
+        alice.set_config(Config::Bot, Some("1")).await?;
+        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
+        assert!(result.is_ok());
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
@@ -5038,6 +5097,32 @@ mod tests {
         Ok(())
     }
 
+    /// Tests that if member added message is completely lost,
+    /// member is eventually added.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_lost_member_added() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        let alice_chat_id = alice
+            .create_group_with_members(ProtectionStatus::Unprotected, "Group", &[bob])
+            .await;
+        let alice_sent = alice.send_text(alice_chat_id, "Hi!").await;
+        let bob_chat_id = bob.recv_msg(&alice_sent).await.chat_id;
+        assert_eq!(get_chat_contacts(bob, bob_chat_id).await?.len(), 2);
+
+        // Attempt to add member, but message is lost.
+        let claire_id = Contact::create(alice, "", "claire@foo.de").await?;
+        add_contact_to_chat(alice, alice_chat_id, claire_id).await?;
+        alice.pop_sent_msg().await;
+
+        let alice_sent = alice.send_text(alice_chat_id, "Hi again!").await;
+        bob.recv_msg(&alice_sent).await;
+        assert_eq!(get_chat_contacts(bob, bob_chat_id).await?.len(), 3);
+
+        Ok(())
+    }
+
     /// Test that group updates are robust to lost messages and eventual out of order arrival.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_modify_chat_lost() -> Result<()> {

src/config.rs

@@ -1007,6 +1007,15 @@ mod tests {
             .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
         sync(&alice0, &alice1).await;
+        // There was a bug that a sync message creates the self-chat with the user avatar instead of
+        // the special icon and that remains so when the self-chat becomes user-visible. Let's check
+        // this.
+        let self_chat = alice0.get_self_chat().await;
+        let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
+        assert_eq!(
+            self_chat_avatar_path,
+            alice0.get_blobdir().join("icon-saved-messages.png")
+        );
         assert!(alice1
             .get_config(Config::Selfavatar)
             .await?

src/contact.rs

@@ -20,7 +20,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{ChatId, ProtectionStatus};
+use crate::chat::{ChatId, ChatIdBlocked, ProtectionStatus};
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -1315,7 +1315,9 @@ impl Contact {
     pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
         let contact_id = self.id;
 
-        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
+        if let Some(ChatIdBlocked { id: chat_id, .. }) =
+            ChatIdBlocked::lookup_by_contact(context, contact_id).await?
+        {
             Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
         } else {
             // 1:1 chat does not exist.

src/decrypt.rs

@@ -57,11 +57,7 @@ pub(crate) async fn prepare_decryption(
             autocrypt_header: None,
             peerstate: None,
             message_time,
-            dkim_results: DkimResults {
-                dkim_passed: false,
-                dkim_should_work: false,
-                allow_keychange: true,
-            },
+            dkim_results: DkimResults { dkim_passed: false },
         });
     }
 
@@ -86,15 +82,13 @@ pub(crate) async fn prepare_decryption(
         None
     };
 
-    let dkim_results = handle_authres(context, mail, from, message_time).await?;
+    let dkim_results = handle_authres(context, mail, from).await?;
     let allow_aeap = get_encrypted_mime(mail).is_some();
     let peerstate = get_autocrypt_peerstate(
         context,
         from,
         autocrypt_header.as_ref(),
         message_time,
-        // Disallowing keychanges is disabled for now:
-        true, // dkim_results.allow_keychange,
         allow_aeap,
     )
     .await?;
@@ -287,19 +281,15 @@ pub(crate) fn keyring_from_peerstate(peerstate: Option<&Peerstate>) -> Vec<Signe
 /// If we already know this fingerprint from another contact's peerstate, return that
 /// peerstate in order to make AEAP work, but don't save it into the db yet.
 ///
-/// The param `allow_change` is used to prevent the autocrypt key from being changed
-/// if we suspect that the message may be forged and have a spoofed sender identity.
-///
 /// Returns updated peerstate.
 pub(crate) async fn get_autocrypt_peerstate(
     context: &Context,
     from: &str,
     autocrypt_header: Option<&Aheader>,
     message_time: i64,
-    allow_change: bool,
     allow_aeap: bool,
 ) -> Result<Option<Peerstate>> {
-    let allow_change = allow_change && !context.is_self_addr(from).await?;
+    let allow_change = !context.is_self_addr(from).await?;
     let mut peerstate;
 
     // Apply Autocrypt header

src/ephemeral.rs

@@ -74,12 +74,13 @@ use async_channel::Receiver;
 use serde::{Deserialize, Serialize};
 use tokio::time::timeout;
 
-use crate::chat::{send_msg, ChatId};
+use crate::chat::{send_msg, ChatId, ChatIdBlocked};
 use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
+use crate::location;
 use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
@@ -348,16 +349,16 @@ pub(crate) async fn start_ephemeral_timers_msgids(
 /// Selects messages which are expired according to
 /// `delete_device_after` setting or `ephemeral_timestamp` column.
 ///
-/// For each message a row ID, chat id and viewtype is returned.
+/// For each message a row ID, chat id, viewtype and location ID is returned.
 async fn select_expired_messages(
     context: &Context,
     now: i64,
-) -> Result<Vec<(MsgId, ChatId, Viewtype)>> {
+) -> Result<Vec<(MsgId, ChatId, Viewtype, u32)>> {
     let mut rows = context
         .sql
         .query_map(
             r#"
-SELECT id, chat_id, type
+SELECT id, chat_id, type, location_id
 FROM msgs
 WHERE
   ephemeral_timestamp != 0
@@ -369,18 +370,21 @@ WHERE
                 let id: MsgId = row.get("id")?;
                 let chat_id: ChatId = row.get("chat_id")?;
                 let viewtype: Viewtype = row.get("type")?;
-                Ok((id, chat_id, viewtype))
+                let location_id: u32 = row.get("location_id")?;
+                Ok((id, chat_id, viewtype, location_id))
             },
             |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
         )
         .await?;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
             .await?
+            .map(|c| c.id)
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE)
             .await?
+            .map(|c| c.id)
             .unwrap_or_default();
 
         let threshold_timestamp = now.saturating_sub(delete_device_after);
@@ -389,7 +393,7 @@ WHERE
             .sql
             .query_map(
                 r#"
-SELECT id, chat_id, type
+SELECT id, chat_id, type, location_id
 FROM msgs
 WHERE
   timestamp < ?1
@@ -408,7 +412,8 @@ WHERE
                     let id: MsgId = row.get("id")?;
                     let chat_id: ChatId = row.get("chat_id")?;
                     let viewtype: Viewtype = row.get("type")?;
-                    Ok((id, chat_id, viewtype))
+                    let location_id: u32 = row.get("location_id")?;
+                    Ok((id, chat_id, viewtype, location_id))
                 },
                 |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
             )
@@ -439,7 +444,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
 
                 // If you change which information is removed here, also change MsgId::trash() and
                 // which information receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-                for (msg_id, chat_id, viewtype) in rows {
+                for (msg_id, chat_id, viewtype, location_id) in rows {
                     transaction.execute(
                         "UPDATE msgs
                      SET chat_id=?, txt='', subject='', txt_raw='',
@@ -448,6 +453,13 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
                         (DC_CHAT_ID_TRASH, msg_id),
                     )?;
 
+                    if location_id > 0 {
+                        transaction.execute(
+                            "DELETE FROM locations WHERE independent=1 AND id=?",
+                            (location_id,),
+                        )?;
+                    }
+
                     msgs_changed.push((chat_id, msg_id));
                     if viewtype == Viewtype::Webxdc {
                         webxdc_deleted.push(msg_id)
@@ -480,11 +492,13 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
 /// `delete_device_after` setting being set.
 async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
             .await?
+            .map(|c| c.id)
             .unwrap_or_default();
-        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE)
             .await?
+            .map(|c| c.id)
             .unwrap_or_default();
 
         let oldest_message_timestamp: Option<i64> = context
@@ -592,6 +606,11 @@ pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiv
             .await
             .log_err(context)
             .ok();
+
+        location::delete_expired(context, time())
+            .await
+            .log_err(context)
+            .ok();
     }
 }
 
@@ -671,8 +690,10 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::download::DownloadState;
+    use crate::location;
+    use crate::message::markseen_msgs;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{TestContext, TestContextManager};
     use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::{
         chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
@@ -1349,4 +1370,44 @@ mod tests {
 
         Ok(())
     }
+
+    /// Tests that POI location is deleted when ephemeral message expires.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_ephemeral_poi_location() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        let chat = alice.create_chat(bob).await;
+
+        let duration = 60;
+        chat.id
+            .set_ephemeral_timer(alice, Timer::Enabled { duration })
+            .await?;
+        let sent = alice.pop_sent_msg().await;
+        bob.recv_msg(&sent).await;
+
+        let mut poi_msg = Message::new(Viewtype::Text);
+        poi_msg.text = "Here".to_string();
+        poi_msg.set_location(10.0, 20.0);
+
+        let alice_sent_message = alice.send_msg(chat.id, &mut poi_msg).await;
+        let bob_received_message = bob.recv_msg(&alice_sent_message).await;
+        markseen_msgs(bob, vec![bob_received_message.id]).await?;
+
+        for account in [alice, bob] {
+            let locations = location::get_range(account, None, None, 0, 0).await?;
+            assert_eq!(locations.len(), 1);
+        }
+
+        SystemTime::shift(Duration::from_secs(100));
+
+        for account in [alice, bob] {
+            delete_expired_messages(account, time()).await?;
+            let locations = location::get_range(account, None, None, 0, 0).await?;
+            assert_eq!(locations.len(), 0);
+        }
+
+        Ok(())
+    }
 }

src/location.rs

@@ -1,4 +1,14 @@
 //! Location handling.
+//!
+//! Delta Chat handles two kind of locations.
+//!
+//! There are two kinds of locations:
+//! - Independent locations, also known as Points of Interest (POI).
+//! - Path locations.
+//!
+//! Locations are sent as KML attachments.
+//! Independent locations are sent in `message.kml` attachments
+//! and path locations are sent in `location.kml` attachments.
 
 use std::time::Duration;
 
@@ -8,6 +18,7 @@ use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 use tokio::time::timeout;
 
 use crate::chat::{self, ChatId};
+use crate::constants::DC_CHAT_ID_TRASH;
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::events::EventType;
@@ -350,6 +361,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         )
         .await?;
 
+    let mut stored_location = false;
     for chat_id in chats {
         context.sql.execute(
                 "INSERT INTO locations  \
@@ -362,6 +374,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                     chat_id,
                     ContactId::SELF,
                 )).await.context("Failed to store location")?;
+        stored_location = true;
 
         info!(context, "Stored location for chat {chat_id}.");
         continue_streaming = true;
@@ -369,6 +382,10 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
     if continue_streaming {
         context.emit_location_changed(Some(ContactId::SELF)).await?;
     };
+    if stored_location {
+        // Interrupt location loop so it may send a location-only message.
+        context.scheduler.interrupt_location().await;
+    }
 
     Ok(continue_streaming)
 }
@@ -461,6 +478,58 @@ pub async fn delete_all(context: &Context) -> Result<()> {
     Ok(())
 }
 
+/// Deletes expired locations.
+///
+/// Only path locations are deleted.
+/// POIs should be deleted when corresponding message is deleted.
+pub(crate) async fn delete_expired(context: &Context, now: i64) -> Result<()> {
+    let Some(delete_device_after) = context.get_config_delete_device_after().await? else {
+        return Ok(());
+    };
+
+    let threshold_timestamp = now.saturating_sub(delete_device_after);
+    let deleted = context
+        .sql
+        .execute(
+            "DELETE FROM locations WHERE independent=0 AND timestamp < ?",
+            (threshold_timestamp,),
+        )
+        .await?
+        > 0;
+    if deleted {
+        info!(context, "Deleted {deleted} expired locations.");
+        context.emit_location_changed(None).await?;
+    }
+    Ok(())
+}
+
+/// Deletes location if it is an independent location.
+///
+/// This function is used when a message is deleted
+/// that has a corresponding `location_id`.
+pub(crate) async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "DELETE FROM locations WHERE independent = 1 AND id=?",
+            (location_id as i32,),
+        )
+        .await?;
+    Ok(())
+}
+
+/// Deletes POI locations that don't have corresponding message anymore.
+pub(crate) async fn delete_orphaned_poi_locations(context: &Context) -> Result<()> {
+    context.sql.execute("
+    DELETE FROM locations
+    WHERE independent=1 AND id NOT IN
+    (SELECT location_id from MSGS LEFT JOIN locations
+     ON locations.id=location_id
+     WHERE location_id>0 -- This check makes the query faster by not looking for locations with ID 0 that don't exist.
+     AND msgs.chat_id != ?)", (DC_CHAT_ID_TRASH,)).await?;
+    Ok(())
+}
+
 /// Returns `location.kml` contents.
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<Option<(String, u32)>> {
     let mut last_added_location_id = 0;
@@ -815,8 +884,10 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::config::Config;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{TestContext, TestContextManager};
+    use crate::tools::SystemTime;
 
     #[test]
     fn test_kml_parse() {
@@ -966,6 +1037,8 @@ Content-Disposition: attachment; filename="location.kml"
         let mut msg = Message::new(Viewtype::Image);
         msg.set_file(file.to_str().unwrap(), None);
         let sent = alice.send_msg(alice_chat.id, &mut msg).await;
+        let alice_msg = Message::load_from_db(&alice, sent.sender_msg_id).await?;
+        assert_eq!(alice_msg.has_location(), false);
 
         let msg = bob.recv_msg_opt(&sent).await.unwrap();
         assert!(msg.chat_id == bob_chat_id);
@@ -974,6 +1047,60 @@ Content-Disposition: attachment; filename="location.kml"
         let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.first().unwrap()).await?;
         assert_eq!(bob_msg.chat_id, bob_chat_id);
         assert_eq!(bob_msg.viewtype, Viewtype::Image);
+        assert_eq!(bob_msg.has_location(), false);
+
+        let bob_locations = get_range(&bob, None, None, 0, 0).await?;
+        assert_eq!(bob_locations.len(), 1);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_delete_expired_locations() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        // Alice enables deletion of messages from device after 1 week.
+        alice
+            .set_config(Config::DeleteDeviceAfter, Some("604800"))
+            .await?;
+        // Bob enables deletion of messages from device after 1 day.
+        bob.set_config(Config::DeleteDeviceAfter, Some("86400"))
+            .await?;
+
+        let alice_chat = alice.create_chat(bob).await;
+
+        // Alice enables location streaming.
+        // Bob receives a message saying that Alice enabled location streaming.
+        send_locations_to_chat(alice, alice_chat.id, 60).await?;
+        bob.recv_msg(&alice.pop_sent_msg().await).await;
+
+        // Alice gets new location from GPS.
+        assert_eq!(set(alice, 10.0, 20.0, 1.0).await?, true);
+        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
+
+        // 10 seconds later location sending stream manages to send location.
+        SystemTime::shift(Duration::from_secs(10));
+        delete_expired(alice, time()).await?;
+        maybe_send_locations(alice).await?;
+        bob.recv_msg_opt(&alice.pop_sent_msg().await).await;
+        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
+        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 1);
+
+        // Day later Bob removes location.
+        SystemTime::shift(Duration::from_secs(86400));
+        delete_expired(alice, time()).await?;
+        delete_expired(bob, time()).await?;
+        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
+        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 0);
+
+        // Week late Alice removes location.
+        SystemTime::shift(Duration::from_secs(604800));
+        delete_expired(alice, time()).await?;
+        delete_expired(bob, time()).await?;
+        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 0);
+        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 0);
 
         Ok(())
     }

src/message.rs

@@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize};
 use tokio::{fs, io};
 
 use crate::blob::BlobObject;
-use crate::chat::{Chat, ChatId};
+use crate::chat::{Chat, ChatId, ChatIdBlocked};
 use crate::chatlist_events;
 use crate::config::Config;
 use crate::constants::{
@@ -22,6 +22,7 @@ use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::imap::markseen_on_imap_table;
+use crate::location::delete_poi_location;
 use crate::mimeparser::{parse_message_id, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
@@ -654,13 +655,11 @@ impl Message {
         Ok(())
     }
 
-    /// Check if a message has a location bound to it.
-    /// These messages are also returned by get_locations()
-    /// and the UI may decide to display a special icon beside such messages,
+    /// Check if a message has a POI location bound to it.
+    /// These locations are also returned by [`location::get_range()`].
+    /// The UI may decide to display a special icon beside such messages.
     ///
-    /// @memberof Message
-    /// @param msg The message object.
-    /// @return 1=Message has location bound to it, 0=No location bound to message.
+    /// [`location::get_range()`]: crate::location::get_range
     pub fn has_location(&self) -> bool {
         self.location_id != 0
     }
@@ -670,13 +669,17 @@ impl Message {
     /// at a position different from the self-location.
     /// You should not call this function
     /// if you want to bind the current self-location to a message;
-    /// this is done by set_location() and send_locations_to_chat().
+    /// this is done by [`location::set()`] and [`send_locations_to_chat()`].
     ///
-    /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
-    /// contact_id set to ContactId::SELF.
+    /// Typically results in the event [`LocationChanged`] with
+    /// `contact_id` set to [`ContactId::SELF`].
     ///
-    /// @param latitude North-south position of the location.
-    /// @param longitude East-west position of the location.
+    /// `latitude` is the North-south position of the location.
+    /// `longitutde` is the East-west position of the location.
+    ///
+    /// [`location::set()`]: crate::location::set
+    /// [`send_locations_to_chat()`]: crate::location::send_locations_to_chat
+    /// [`LocationChanged`]: crate::events::EventType::LocationChanged
     pub fn set_location(&mut self, latitude: f64, longitude: f64) {
         if latitude == 0.0 && longitude == 0.0 {
             return;
@@ -1571,17 +1574,6 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
-async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
-    context
-        .sql
-        .execute(
-            "DELETE FROM locations WHERE independent = 1 AND id=?;",
-            (location_id as i32,),
-        )
-        .await?;
-    Ok(())
-}
-
 /// Marks requested messages as seen.
 pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()> {
     if msg_ids.is_empty() {
@@ -1821,8 +1813,9 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
+    let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
         .await?
+        .map(|c| c.id)
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
 

src/mimeparser.rs

@@ -415,8 +415,6 @@ impl MimeMessage {
             if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
                 if timestamp_sent > peerstate.last_seen_autocrypt
                     && mail.ctype.mimetype != "multipart/report"
-                // Disallowing keychanges is disabled for now:
-                // && decryption_info.dkim_results.allow_keychange
                 {
                     peerstate.degrade_encryption(timestamp_sent);
                 }
@@ -506,13 +504,6 @@ impl MimeMessage {
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context).await?;
 
-        // Disallowing keychanges is disabled for now
-        // if !decryption_info.dkim_results.allow_keychange {
-        //     for part in parser.parts.iter_mut() {
-        //         part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string());
-        //     }
-        // }
-
         if parser.is_mime_modified {
             parser.decoded_data = mail_raw;
         }

src/receive_imf.rs

@@ -1669,11 +1669,10 @@ async fn save_locations(
         if let Some(addr) = &location_kml.addr {
             let contact = Contact::get_by_id(context, from_id).await?;
             if contact.get_addr().to_lowercase() == addr.to_lowercase() {
-                if let Some(newest_location_id) =
-                    location::save(context, chat_id, from_id, &location_kml.locations, false)
-                        .await?
+                if location::save(context, chat_id, from_id, &location_kml.locations, false)
+                    .await?
+                    .is_some()
                 {
-                    location::set_msg_location_id(context, msg_id, newest_location_id).await?;
                     send_event = true;
                 }
             } else {

src/sql.rs

@@ -15,6 +15,7 @@ use crate::context::Context;
 use crate::debug_logging::set_debug_logging_xdc;
 use crate::ephemeral::start_ephemeral_timers;
 use crate::imex::BLOBS_BACKUP_NAME;
+use crate::location::delete_orphaned_poi_locations;
 use crate::log::LogExt;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::param::{Param, Params};
@@ -779,6 +780,14 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
+    // Delete POI locations
+    // which don't have corresponding message.
+    delete_orphaned_poi_locations(context)
+        .await
+        .context("Failed to delete orphaned POI locations")
+        .log_err(context)
+        .ok();
+
     info!(context, "Housekeeping done.");
     Ok(())
 }

src/sql/migrations.rs

@@ -613,6 +613,7 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         ).await?;
     }
     if dbversion < 93 {
+        // `sending_domains` is now unused, but was not removed for backwards compatibility.
         sql.execute_migration(
             "CREATE TABLE sending_domains(domain TEXT PRIMARY KEY, dkim_works INTEGER DEFAULT 0);",
             93,

src/tools.rs

@@ -735,7 +735,7 @@ Message-ID: 2dfdbde7@example.org
 Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
 Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000
 
-DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
+DKIM Results: Passed=true";
         check_parse_receive_headers_integration(raw, expected).await;
 
         let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
@@ -754,7 +754,7 @@ Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0
 Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
 Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
 
-DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
+DKIM Results: Passed=true";
         check_parse_receive_headers_integration(raw, expected).await;
     }