fix(node): undefine NAPI_EXPERIMENTAL

This fixes build with Node v20.12.2.

Related upstream issue: <https://github.com/nodejs/node/issues/52229>

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.78.0
+      RUSTUP_TOOLCHAIN: 1.77.1
     steps:
       - uses: actions/checkout@v4
         with:
@@ -83,11 +83,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.78.0
+            rust: 1.77.1
           - os: windows-latest
-            rust: 1.77.0
+            rust: 1.77.1
           - os: macos-latest
-            rust: 1.78.0
+            rust: 1.77.1
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest

.github/workflows/deltachat-rpc-server.yml

@@ -266,128 +266,3 @@ jobs:
       - name: Publish deltachat-rpc-client to PyPI
         if: github.event_name == 'release'
         uses: pypa/gh-action-pypi-publish@release/v1
-
-  publish_npm_package:
-    name: Build & Publish npm prebuilds and deltachat-rpc-server
-    needs: ["build_linux", "build_windows", "build_macos"]
-    runs-on: "ubuntu-latest"
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          show-progress: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.11' 
-
-      - name: Download Linux aarch64 binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-aarch64-linux
-          path: deltachat-rpc-server-aarch64-linux.d
-
-      - name: Download Linux armv7l binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-armv7l-linux
-          path: deltachat-rpc-server-armv7l-linux.d
-
-      - name: Download Linux armv6l binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-armv6l-linux
-          path: deltachat-rpc-server-armv6l-linux.d
-
-      - name: Download Linux i686 binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-i686-linux
-          path: deltachat-rpc-server-i686-linux.d
-
-      - name: Download Linux x86_64 binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-x86_64-linux
-          path: deltachat-rpc-server-x86_64-linux.d
-
-      - name: Download Win32 binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-win32
-          path: deltachat-rpc-server-win32.d
-
-      - name: Download Win64 binary
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-win64
-          path: deltachat-rpc-server-win64.d
-
-      - name: Download macOS binary for x86_64
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-x86_64-macos
-          path: deltachat-rpc-server-x86_64-macos.d
-
-      - name: Download macOS binary for aarch64
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-aarch64-macos
-          path: deltachat-rpc-server-aarch64-macos.d
-
-      - name: Download Android binary for arm64-v8a
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-arm64-v8a-android
-          path: deltachat-rpc-server-arm64-v8a-android.d
-
-      - name: Download Android binary for armeabi-v7a
-        uses: actions/download-artifact@v4
-        with:
-          name: deltachat-rpc-server-armeabi-v7a-android
-          path: deltachat-rpc-server-armeabi-v7a-android.d
-
-      - name: make npm packets for prebuilds and `@deltachat/stdio-rpc-server`
-        run: |
-          cd deltachat-rpc-server/npm-package
-
-          python --version
-
-          python scripts/pack_binary_for_platform.py aarch64-unknown-linux-musl     ../../deltachat-rpc-server-aarch64-linux.d/deltachat-rpc-server
-          python scripts/pack_binary_for_platform.py armv7-unknown-linux-musleabihf ../../deltachat-rpc-server-armv7l-linux.d/deltachat-rpc-server 
-          python scripts/pack_binary_for_platform.py arm-unknown-linux-musleabihf   ../../deltachat-rpc-server-armv6l-linux.d/deltachat-rpc-server 
-          python scripts/pack_binary_for_platform.py i686-unknown-linux-musl        ../../deltachat-rpc-server-i686-linux.d/deltachat-rpc-server 
-          python scripts/pack_binary_for_platform.py x86_64-unknown-linux-musl      ../../deltachat-rpc-server-x86_64-linux.d/deltachat-rpc-server
-          python scripts/pack_binary_for_platform.py i686-pc-windows-gnu            ../../deltachat-rpc-server-win32.d/deltachat-rpc-server.exe 
-          python scripts/pack_binary_for_platform.py x86_64-pc-windows-gnu          ../../deltachat-rpc-server-win64.d/deltachat-rpc-server.exe 
-          python scripts/pack_binary_for_platform.py x86_64-apple-darwin            ../../deltachat-rpc-server-x86_64-macos.d/deltachat-rpc-server 
-          python scripts/pack_binary_for_platform.py aarch64-apple-darwin           ../../deltachat-rpc-server-aarch64-macos.d/deltachat-rpc-server 
-          python scripts/pack_binary_for_platform.py aarch64-linux-android          ../../deltachat-rpc-server-arm64-v8a-android.d/deltachat-rpc-server
-          python scripts/pack_binary_for_platform.py armv7-linux-androideabi        ../../deltachat-rpc-server-armeabi-v7a-android.d/deltachat-rpc-server
-
-          ls -lah platform_package
-
-          for platform in ./platform_package/*; do npm pack "$platform"; done
-          npm pack    
-          ls -lah
-      
-      - name: Upload to artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: deltachat-rpc-server-npm-package
-          path: deltachat-rpc-server/npm-package/*.tgz
-          if-no-files-found: error
-
-      - name: Upload npm packets to the GitHub release
-        if: github.event_name == 'release'
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-        run: |
-          gh release upload ${{ github.ref_name }} \
-            --repo ${{ github.repository }} \
-            *.tgz
-
-      - name: Publish npm packets for prebuilds and `@deltachat/stdio-rpc-server`
-        if: github.event_name == 'release'
-        run: |
-          ls -lah platform_package
-
-          for platform in *.tgz; do npm publish "$platform"; done

.github/workflows/node-tests.yml

@@ -21,14 +21,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-14, windows-latest]
+        node: ["18", "20"]
     steps:
       - uses: actions/checkout@v4
         with:
           show-progress: false
       - uses: actions/setup-node@v4
         with:
-          node-version: "18"
+          node-version: ${{ matrix.node }}
+      - if: matrix.os == 'macos-14'
+        run: sudo xcode-select --switch /Applications/Xcode_15.3.app/Contents/Developer
       - name: System info
         run: |
           rustc -vV

.github/workflows/upload-docs.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - build_jsonrpc_docs_ci
 
 jobs:
   build-rs:
@@ -18,11 +17,13 @@ jobs:
         run: |
           cargo doc --package deltachat --no-deps --document-private-items
       - name: Upload to rs.delta.chat
-        run: |
-          mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
-          chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/target/doc "${{ secrets.USERNAME }}@rs.delta.chat:/var/www/html/rs/"
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: ${{ secrets.USERNAME }}
+          KEY: ${{ secrets.KEY }}
+          HOST: "delta.chat"
+          SOURCE: "target/doc"
+          TARGET: "/var/www/html/rs/"
 
   build-python:
     runs-on: ubuntu-latest
@@ -61,31 +62,3 @@ jobs:
           echo "${{ secrets.CODESPEAK_KEY }}" > "$HOME/.ssh/key"
           chmod 600 "$HOME/.ssh/key"
           rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/result/html/ "delta@c.delta.chat:/home/delta/build-c/master"
-
-  build-ts:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./deltachat-jsonrpc/typescript
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          show-progress: false
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Use Node.js
-        uses: actions/setup-node@v2
-        with:
-          node-version: '18'
-      - name: npm install
-        run: npm install
-      - name: npm run build
-        run: npm run build
-      - name: Run docs script
-        run: npm run docs
-      - name: Upload to js.jsonrpc.delta.chat
-        run: |
-          mkdir -p "$HOME/.ssh"
-          echo "${{ secrets.KEY }}" > "$HOME/.ssh/key"
-          chmod 600 "$HOME/.ssh/key"
-          rsync -avzh -e "ssh -i $HOME/.ssh/key -o StrictHostKeyChecking=no" $GITHUB_WORKSPACE/deltachat-jsonrpc/typescript/docs/ "${{ secrets.USERNAME }}@js.jsonrpc.delta.chat:/var/www/html/js-jsonrpc/"

.gitignore

@@ -33,7 +33,6 @@ deltachat-ffi/xml
 
 coverage/
 .DS_Store
-.vscode
 .vscode/launch.json
 python/accounts.txt
 python/all-testaccounts.txt

CHANGELOG.md

@@ -1,48 +1,5 @@
 # Changelog
 
-## [1.137.4] - 2024-04-24
-
-### API-Changes
-
-- [**breaking**] Remove `Stream` implementation for `EventEmitter`.
-- Experimental Webxdc Integration API, Maps Integration ([#5461](https://github.com/deltachat/deltachat-core-rust/pull/5461)).
-
-### Features / Changes
-
-- Add progressive backoff for failing IMAP connection attempts ([#5443](https://github.com/deltachat/deltachat-core-rust/pull/5443)).
-- Replace event channel with broadcast channel.
-- Mark contact request messages as seen on IMAP.
-
-### Fixes
-
-- Convert images to RGB8 (without alpha) before encoding into JPEG to fix sending of large RGBA images.
-- Don't set `is_bot` for webxdc status updates ([#5445](https://github.com/deltachat/deltachat-core-rust/pull/5445)).
-- Do not fail if Autocrypt Setup Message has no encryption preference to fix key transfer from K-9 Mail to Delta Chat.
-- Use only CRLF in Autocrypt Setup Message.
-- python: Use cached message object if `dc_get_msg()` returns `NULL`.
-- python: `Message::is_outgoing`: Don't reload message from db.
-- python: `_map_ffi_event`: Always check if `get_message_by_id()` returned None.
-- node: Undefine `NAPI_EXPERIMENTAL` to fix build with new clang.
-
-### Build system
-
-- nix: Add `imap-tools` as `deltachat-rpc-client` dependency.
-- nix: Add `./deltachat-contact-tools` to sources.
-- nix: Update nix flake.
-- deps: Update rustls to 0.21.11.
-
-### Documentation
-
-- Update references to SecureJoin protocols.
-- Fix broken references in documentation comments.
-
-### Refactor
-
-- imap: remove `RwLock` from `ratelimit`.
-- deltachat-ffi: Remove unused `ResultNullableExt`.
-- Remove duplicate clippy exceptions.
-- Group `use` at the top of the test modules.
-
 ## [1.137.3] - 2024-04-16
 
 ### API-Changes
@@ -3982,4 +3939,3 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.137.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.137.0...v1.137.1
 [1.137.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.137.1...v1.137.2
 [1.137.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.137.2...v1.137.3
-[1.137.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.137.3...v1.137.4

Cargo.lock

@@ -163,9 +163,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.82"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
+checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
 dependencies = [
  "backtrace",
 ]
@@ -252,9 +252,9 @@ dependencies = [
 
 [[package]]
 name = "async-channel"
-version = "2.2.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "136d4d23bcc79e27423727b36823d86233aad06dfea531837b038394d11e9928"
+checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3"
 dependencies = [
  "concurrent-queue",
  "event-listener 5.2.0",
@@ -282,7 +282,7 @@ version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "98892ebee4c05fc66757e600a7466f0d9bfcde338f645d64add323789f26cb36"
 dependencies = [
- "async-channel 2.2.1",
+ "async-channel 2.2.0",
  "base64 0.21.7",
  "bytes",
  "chrono",
@@ -344,7 +344,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -478,9 +478,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64"
-version = "0.22.1"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
 
 [[package]]
 name = "base64ct"
@@ -724,9 +724,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.38"
+version = "0.4.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
@@ -1059,7 +1059,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -1134,9 +1134,9 @@ dependencies = [
 
 [[package]]
 name = "data-encoding"
-version = "2.6.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
+checksum = "7e962a19be5cfc3f3bf6dd8f61eb50107f356ad6270fbb3ed41476571db78be5"
 
 [[package]]
 name = "default-net"
@@ -1157,18 +1157,18 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.137.4"
+version = "1.137.3"
 dependencies = [
  "ansi_term",
  "anyhow",
  "async-broadcast",
- "async-channel 2.2.1",
+ "async-channel 2.2.0",
  "async-imap",
  "async-native-tls",
  "async-smtp",
  "async_zip",
  "backtrace",
- "base64 0.22.1",
+ "base64 0.22.0",
  "brotli",
  "chrono",
  "criterion",
@@ -1239,10 +1239,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat-contact-tools"
-version = "0.0.0"
+version = "0.1.0"
 dependencies = [
  "anyhow",
- "chrono",
  "once_cell",
  "regex",
  "rusqlite",
@@ -1250,12 +1249,12 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.137.4"
+version = "1.137.3"
 dependencies = [
  "anyhow",
- "async-channel 2.2.1",
+ "async-channel 2.2.0",
  "axum",
- "base64 0.22.1",
+ "base64 0.22.0",
  "deltachat",
  "env_logger 0.11.3",
  "futures",
@@ -1274,7 +1273,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "1.137.4"
+version = "1.137.3"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1289,7 +1288,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.137.4"
+version = "1.137.3"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1313,12 +1312,12 @@ name = "deltachat_derive"
 version = "2.0.0"
 dependencies = [
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.137.4"
+version = "1.137.3"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1522,7 +1521,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -1831,7 +1830,7 @@ dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -1844,7 +1843,7 @@ dependencies = [
  "num-traits",
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -2165,7 +2164,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -2639,9 +2638,9 @@ dependencies = [
 
 [[package]]
 name = "imap-proto"
-version = "0.16.5"
+version = "0.16.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de555d9526462b6f9ece826a26fb7c67eca9a0245bd9ff84fa91972a5d5d8856"
+checksum = "22e70cd66882c8cb1c9802096ba75212822153c51478dc61621e1a22f6c92361"
 dependencies = [
  "nom",
 ]
@@ -2907,9 +2906,9 @@ dependencies = [
 
 [[package]]
 name = "mailparse"
-version = "0.15.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3da03d5980411a724e8aaf7b61a7b5e386ec55a7fb49ee3d0ff79efc7e5e7c7e"
+checksum = "2d096594926cab442e054e047eb8c1402f7d5b2272573b97ba68aa40629f9757"
 dependencies = [
  "charset",
  "data-encoding",
@@ -3167,7 +3166,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -3228,7 +3227,7 @@ dependencies = [
  "proc-macro-crate",
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -3290,7 +3289,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -3531,7 +3530,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -3748,9 +3747,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.81"
+version = "1.0.78"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba"
+checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
 dependencies = [
  "unicode-ident",
 ]
@@ -3861,9 +3860,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
  "proc-macro2",
 ]
@@ -4395,9 +4394,9 @@ dependencies = [
 
 [[package]]
 name = "schemars"
-version = "0.8.19"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc6e7ed6919cb46507fb01ff1654309219f62b4d603822501b0b80d42f6f21ef"
+checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29"
 dependencies = [
  "dyn-clone",
  "schemars_derive",
@@ -4407,14 +4406,14 @@ dependencies = [
 
 [[package]]
 name = "schemars_derive"
-version = "0.8.19"
+version = "0.8.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "185f2b7aa7e02d418e453790dde16890256bbd2bcd04b7dc5348811052b53f49"
+checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967"
 dependencies = [
  "proc-macro2",
  "quote",
  "serde_derive_internals",
- "syn 2.0.60",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -4501,9 +4500,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.200"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f"
+checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
 dependencies = [
  "serde_derive",
 ]
@@ -4528,24 +4527,24 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.200"
+version = "1.0.197"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb"
+checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
 name = "serde_derive_internals"
-version = "0.29.0"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "330f01ce65a3a5fe59a60c82f3c9a024b573b8a6e875bd233fe5f934e71d54e3"
+checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -4819,7 +4818,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -4841,9 +4840,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.60"
+version = "2.0.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3"
+checksum = "11a6ae1e52eb25aab8f3fb9fca13be982a373b8f1157ca14b897a825ba4a2d35"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4963,22 +4962,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.59"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa"
+checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.59"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66"
+checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -5095,7 +5094,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -5268,7 +5267,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -5562,7 +5561,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
  "wasm-bindgen-shared",
 ]
 
@@ -5596,7 +5595,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -5992,7 +5991,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]
@@ -6012,7 +6011,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.60",
+ "syn 2.0.57",
 ]
 
 [[package]]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.137.4"
+version = "1.137.3"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.77"
@@ -40,7 +40,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
 async-broadcast = "0.7.0"
-async-channel = "2.2.1"
+async-channel = "2.0.0"
 async-imap = { version = "0.9.7", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
@@ -48,7 +48,7 @@ async_zip = { version = "0.0.12", default-features = false, features = ["deflate
 backtrace = "0.3"
 base64 = "0.22"
 brotli = { version = "5", default-features=false, features = ["std"] }
-chrono = { workspace = true }
+chrono = { version = "0.4.37", default-features=false, features = ["clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
@@ -64,7 +64,7 @@ iroh = { version = "0.4.2", default-features = false }
 kamadak-exif = "0.5.3"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
-mailparse = "0.15"
+mailparse = "0.14"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
@@ -168,8 +168,7 @@ harness = false
 anyhow = "1"
 once_cell = "1.18.0"
 regex = "1.10"
-rusqlite = "0.31"
-chrono = { version = "0.4.38", default-features=false, features = ["alloc", "clock", "std"] }
+rusqlite = { version = "0.31" }
 
 [features]
 default = ["vendored"]

deltachat-contact-tools/Cargo.toml

@@ -1,9 +1,10 @@
 [package]
 name = "deltachat-contact-tools"
-version = "0.0.0" # No semver-stable versioning
+version = "0.1.0"
 edition = "2021"
-description = "Contact-related tools, like parsing vcards and sanitizing name and address. Meant for internal use in the deltachat crate."
+description = "Contact-related tools, like parsing vcards and sanitizing name and address"
 license = "MPL-2.0"
+# TODO maybe it should be called "deltachat-text-utils" or similar?
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -12,7 +13,6 @@ anyhow = { workspace = true }
 once_cell = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true } # Needed in order to `impl rusqlite::types::ToSql for EmailAddress`. Could easily be put behind a feature.
-chrono = { workspace = true }
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.

deltachat-contact-tools/src/lib.rs

@@ -29,181 +29,10 @@ use std::fmt;
 use std::ops::Deref;
 
 use anyhow::bail;
-use anyhow::Context as _;
 use anyhow::Result;
-use chrono::{DateTime, NaiveDateTime};
 use once_cell::sync::Lazy;
 use regex::Regex;
 
-// TODOs to clean up:
-// - Check if sanitizing is done correctly everywhere
-// - Apply lints everywhere (https://doc.rust-lang.org/cargo/reference/workspaces.html#the-lints-table)
-
-#[derive(Debug)]
-/// A Contact, as represented in a VCard.
-pub struct VcardContact {
-    /// The email address, vcard property `email`
-    pub addr: String,
-    /// The contact's display name, vcard property `fn`
-    pub display_name: String,
-    /// The contact's public PGP key in Base64, vcard property `key`
-    pub key: Option<String>,
-    /// The contact's profile image (=avatar) in Base64, vcard property `photo`
-    pub profile_image: Option<String>,
-    /// The timestamp when the vcard was created / last updated, vcard property `rev`
-    pub timestamp: Result<u64>,
-}
-
-/// Returns a vCard containing given contacts.
-///
-/// Calling [`parse_vcard()`] on the returned result is a reverse operation.
-pub fn make_vcard(contacts: &[VcardContact]) -> String {
-    fn format_timestamp(c: &VcardContact) -> Option<String> {
-        let timestamp = *c.timestamp.as_ref().ok()?;
-        let timestamp: i64 = timestamp.try_into().ok()?;
-        let datetime = DateTime::from_timestamp(timestamp, 0)?;
-        Some(datetime.format("%Y%m%dT%H%M%SZ").to_string())
-    }
-
-    let mut res = "".to_string();
-    for c in contacts {
-        let addr = &c.addr;
-        let display_name = match c.display_name.is_empty() {
-            false => &c.display_name,
-            true => &c.addr,
-        };
-        res += &format!(
-            "BEGIN:VCARD\n\
-             VERSION:4.0\n\
-             EMAIL:{addr}\n\
-             FN:{display_name}\n"
-        );
-        if let Some(key) = &c.key {
-            res += &format!("KEY:data:application/pgp-keys;base64,{key}\n");
-        }
-        if let Some(profile_image) = &c.profile_image {
-            res += &format!("PHOTO:data:image/jpeg;base64,{profile_image}\n");
-        }
-        if let Some(timestamp) = format_timestamp(c) {
-            res += &format!("REV:{timestamp}\n");
-        }
-        res += "END:VCARD\n";
-    }
-    res
-}
-
-/// Parses `VcardContact`s from a given `&str`.
-pub fn parse_vcard(vcard: &str) -> Result<Vec<VcardContact>> {
-    fn remove_prefix<'a>(s: &'a str, prefix: &str) -> Option<&'a str> {
-        let start_of_s = s.get(..prefix.len())?;
-
-        if start_of_s.eq_ignore_ascii_case(prefix) {
-            s.get(prefix.len()..)
-        } else {
-            None
-        }
-    }
-    fn vcard_property<'a>(s: &'a str, property: &str) -> Option<&'a str> {
-        let remainder = remove_prefix(s, property)?;
-        // If `s` is `EMAIL;TYPE=work:alice@example.com` and `property` is `EMAIL`,
-        // then `remainder` is now `;TYPE=work:alice@example.com`
-
-        // TODO this doesn't handle the case where there are quotes around a colon
-        let (params, value) = remainder.split_once(':')?;
-        // In the example from above, `params` is now `;TYPE=work`
-        // and `value` is now `alice@example.com`
-
-        if params
-            .chars()
-            .next()
-            .filter(|c| !c.is_ascii_punctuation() || *c == '_')
-            .is_some()
-        {
-            // `s` started with `property`, but the next character after it was not punctuation,
-            // so this line's property is actually something else
-            return None;
-        }
-        Some(value)
-    }
-    fn parse_datetime(datetime: &str) -> Result<u64> {
-        // According to https://www.rfc-editor.org/rfc/rfc6350#section-4.3.5, the timestamp
-        // is in ISO.8601.2004 format. DateTime::parse_from_rfc3339() apparently parses
-        // ISO.8601, but fails to parse any of the examples given.
-        // So, instead just parse using a format string.
-
-        // Parses 19961022T140000Z, 19961022T140000-05, or 19961022T140000-0500.
-        let timestamp = match DateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S%#z") {
-            Ok(datetime) => datetime.timestamp(),
-            // Parses 19961022T140000.
-            Err(e) => match NaiveDateTime::parse_from_str(datetime, "%Y%m%dT%H%M%S") {
-                Ok(datetime) => datetime
-                    .and_local_timezone(chrono::offset::Local)
-                    .single()
-                    .context("Could not apply local timezone to parsed date and time")?
-                    .timestamp(),
-                Err(_) => return Err(e.into()),
-            },
-        };
-        Ok(timestamp.try_into()?)
-    }
-
-    let mut lines = vcard.lines().peekable();
-    let mut contacts = Vec::new();
-
-    while lines.peek().is_some() {
-        // Skip to the start of the vcard:
-        for line in lines.by_ref() {
-            if line.eq_ignore_ascii_case("BEGIN:VCARD") {
-                break;
-            }
-        }
-
-        let mut display_name = None;
-        let mut addr = None;
-        let mut key = None;
-        let mut photo = None;
-        let mut datetime = None;
-
-        for line in lines.by_ref() {
-            if let Some(email) = vcard_property(line, "email") {
-                addr.get_or_insert(email);
-            } else if let Some(name) = vcard_property(line, "fn") {
-                display_name.get_or_insert(name);
-            } else if let Some(k) = remove_prefix(line, "KEY;PGP;ENCODING=BASE64:")
-                .or_else(|| remove_prefix(line, "KEY;TYPE=PGP;ENCODING=b:"))
-                .or_else(|| remove_prefix(line, "KEY:data:application/pgp-keys;base64,"))
-            {
-                key.get_or_insert(k);
-            } else if let Some(p) = remove_prefix(line, "PHOTO;JPEG;ENCODING=BASE64:")
-                .or_else(|| remove_prefix(line, "PHOTO;TYPE=JPEG;ENCODING=b:"))
-                .or_else(|| remove_prefix(line, "PHOTO;ENCODING=BASE64;TYPE=JPEG:"))
-                .or_else(|| remove_prefix(line, "PHOTO:data:image/jpeg;base64,"))
-            {
-                photo.get_or_insert(p);
-            } else if let Some(rev) = vcard_property(line, "rev") {
-                datetime.get_or_insert(rev);
-            } else if line.eq_ignore_ascii_case("END:VCARD") {
-                break;
-            }
-        }
-
-        let (display_name, addr) =
-            sanitize_name_and_addr(display_name.unwrap_or(""), addr.unwrap_or(""));
-
-        contacts.push(VcardContact {
-            display_name,
-            addr,
-            key: key.map(|s| s.to_string()),
-            profile_image: photo.map(|s| s.to_string()),
-            timestamp: datetime
-                .context("No timestamp in vcard")
-                .and_then(parse_datetime),
-        });
-    }
-
-    Ok(contacts)
-}
-
 /// Valid contact address.
 #[derive(Debug, Clone)]
 pub struct ContactAddress(String);
@@ -252,10 +81,14 @@ impl rusqlite::types::ToSql for ContactAddress {
 /// Make the name and address
 pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
-    let (name, addr) = if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+    if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
         (
             if name.is_empty() {
-                strip_rtlo_characters(captures.get(1).map_or("", |m| m.as_str()))
+                strip_rtlo_characters(
+                    &captures
+                        .get(1)
+                        .map_or("".to_string(), |m| normalize_name(m.as_str())),
+                )
             } else {
                 strip_rtlo_characters(name)
             },
@@ -264,21 +97,8 @@ pub fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
                 .map_or("".to_string(), |m| m.as_str().to_string()),
         )
     } else {
-        (
-            strip_rtlo_characters(&normalize_name(name)),
-            addr.to_string(),
-        )
-    };
-    let mut name = normalize_name(&name);
-
-    // If the 'display name' is just the address, remove it:
-    // Otherwise, the contact would sometimes be shown as "alice@example.com (alice@example.com)" (see `get_name_n_addr()`).
-    // If the display name is empty, DC will just show the address when it needs a display name.
-    if name == addr {
-        name = "".to_string();
+        (strip_rtlo_characters(name), addr.to_string())
     }
-
-    (name, addr)
 }
 
 /// Normalize a name.
@@ -410,105 +230,8 @@ impl rusqlite::types::ToSql for EmailAddress {
 
 #[cfg(test)]
 mod tests {
-    use chrono::TimeZone;
-
     use super::*;
 
-    #[test]
-    fn test_vcard_thunderbird() {
-        let contacts = parse_vcard(
-            "BEGIN:VCARD
-VERSION:4.0
-FN:'Alice Mueller'
-EMAIL;PREF=1:alice.mueller@posteo.de
-UID:a8083264-ca47-4be7-98a8-8ec3db1447ca
-END:VCARD
-BEGIN:VCARD
-VERSION:4.0
-FN:'bobzzz@freenet.de'
-EMAIL;PREF=1:bobzzz@freenet.de
-UID:cac4fef4-6351-4854-bbe4-9b6df857eaed
-END:VCARD
-",
-        )
-        .unwrap();
-
-        assert_eq!(contacts[0].addr, "alice.mueller@posteo.de".to_string());
-        assert_eq!(contacts[0].display_name, "Alice Mueller".to_string());
-        assert_eq!(contacts[0].key, None);
-        assert_eq!(contacts[0].profile_image, None);
-        assert!(contacts[0].timestamp.is_err());
-
-        assert_eq!(contacts[1].addr, "bobzzz@freenet.de".to_string());
-        assert_eq!(contacts[1].display_name, "".to_string());
-        assert_eq!(contacts[1].key, None);
-        assert_eq!(contacts[1].profile_image, None);
-        assert!(contacts[1].timestamp.is_err());
-
-        assert_eq!(contacts.len(), 2);
-    }
-
-    #[test]
-    fn test_vcard_simple_example() {
-        let contacts = parse_vcard(
-            "BEGIN:VCARD
-VERSION:4.0
-FN:Alice Wonderland
-N:Wonderland;Alice;;;Ms.
-GENDER:W
-EMAIL;TYPE=work:alice@example.com
-KEY;TYPE=PGP;ENCODING=b:[base64-data]
-REV:20240418T184242Z
-
-END:VCARD",
-        )
-        .unwrap();
-
-        assert_eq!(contacts[0].addr, "alice@example.com".to_string());
-        assert_eq!(contacts[0].display_name, "Alice Wonderland".to_string());
-        assert_eq!(contacts[0].key, Some("[base64-data]".to_string()));
-        assert_eq!(contacts[0].profile_image, None);
-        assert_eq!(*contacts[0].timestamp.as_ref().unwrap(), 1713465762);
-
-        assert_eq!(contacts.len(), 1);
-    }
-
-    #[test]
-    fn test_make_and_parse_vcard() {
-        let contacts = [
-            VcardContact {
-                addr: "alice@example.org".to_string(),
-                display_name: "Alice Wonderland".to_string(),
-                key: Some("[base64-data]".to_string()),
-                profile_image: Some("image in Base64".to_string()),
-                timestamp: Ok(1713465762),
-            },
-            VcardContact {
-                addr: "bob@example.com".to_string(),
-                display_name: "".to_string(),
-                key: None,
-                profile_image: None,
-                timestamp: Ok(0),
-            },
-        ];
-        for len in 0..=contacts.len() {
-            let contacts = &contacts[0..len];
-            let vcard = make_vcard(contacts);
-            let parsed = parse_vcard(&vcard).unwrap();
-            assert_eq!(parsed.len(), contacts.len());
-            for i in 0..parsed.len() {
-                assert_eq!(parsed[i].addr, contacts[i].addr);
-                assert_eq!(parsed[i].display_name, contacts[i].display_name);
-                assert_eq!(parsed[i].key, contacts[i].key);
-                assert_eq!(parsed[i].profile_image, contacts[i].profile_image);
-                assert_eq!(
-                    parsed[i].timestamp.as_ref().unwrap(),
-                    contacts[i].timestamp.as_ref().unwrap()
-                );
-            }
-        }
-    }
-
     #[test]
     fn test_contact_address() -> Result<()> {
         let alice_addr = "alice@example.org";
@@ -554,62 +277,4 @@ END:VCARD",
         assert!(EmailAddress::new("u@tt").is_ok());
         assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
     }
-
-    #[test]
-    fn test_vcard_android() {
-        let contacts = parse_vcard(
-            "BEGIN:VCARD
-VERSION:2.1
-N:;Bob;;;
-FN:Bob
-TEL;CELL:+1-234-567-890
-EMAIL;HOME:bob@example.org
-END:VCARD
-BEGIN:VCARD
-VERSION:2.1
-N:;Alice;;;
-FN:Alice
-EMAIL;HOME:alice@example.org
-END:VCARD
-",
-        )
-        .unwrap();
-
-        assert_eq!(contacts[0].addr, "bob@example.org".to_string());
-        assert_eq!(contacts[0].display_name, "Bob".to_string());
-        assert_eq!(contacts[0].key, None);
-        assert_eq!(contacts[0].profile_image, None);
-
-        assert_eq!(contacts[1].addr, "alice@example.org".to_string());
-        assert_eq!(contacts[1].display_name, "Alice".to_string());
-        assert_eq!(contacts[1].key, None);
-        assert_eq!(contacts[1].profile_image, None);
-
-        assert_eq!(contacts.len(), 2);
-    }
-
-    #[test]
-    fn test_vcard_local_datetime() {
-        let contacts = parse_vcard(
-            "BEGIN:VCARD\n\
-             VERSION:4.0\n\
-             FN:Alice Wonderland\n\
-             EMAIL;TYPE=work:alice@example.org\n\
-             REV:20240418T184242\n\
-             END:VCARD",
-        )
-        .unwrap();
-        assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts[0].addr, "alice@example.org".to_string());
-        assert_eq!(contacts[0].display_name, "Alice Wonderland".to_string());
-        assert_eq!(
-            *contacts[0].timestamp.as_ref().unwrap(),
-            chrono::offset::Local
-                .with_ymd_and_hms(2024, 4, 18, 18, 42, 42)
-                .unwrap()
-                .timestamp()
-                .try_into()
-                .unwrap()
-        );
-    }
 }

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.137.4"
+version = "1.137.3"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -4105,19 +4105,6 @@ char*           dc_msg_get_subject            (const dc_msg_t* msg);
 char*           dc_msg_get_file               (const dc_msg_t* msg);
 
 
-/**
- * Save file copy at the user-provided path.
- *
- * Fails if file already exists at the provided path.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @param path Destination file path with filename and extension.
- * @return 0 on failure, 1 on success.
- */
-int             dc_msg_save_file              (const dc_msg_t* msg, const char* path);
-
-
 /**
  * Get an original attachment filename, with extension but without the path. To get the full path,
  * use dc_msg_get_file().
@@ -4390,9 +4377,9 @@ int             dc_msg_has_deviating_timestamp(const dc_msg_t* msg);
 
 
 /**
- * Check if a message has a POI location bound to it.
- * These locations are also returned by dc_get_locations()
- * The UI may decide to display a special icon beside such messages.
+ * Check if a message has a location bound to it.
+ * These messages are also returned by dc_get_locations()
+ * and the UI may decide to display a special icon beside such messages,
  *
  * @memberof dc_msg_t
  * @param msg The message object.

deltachat-ffi/src/lib.rs

@@ -3368,34 +3368,6 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha
         .unwrap_or_else(|| "".strdup())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_msg_save_file(
-    msg: *mut dc_msg_t,
-    path: *const libc::c_char,
-) -> libc::c_int {
-    if msg.is_null() || path.is_null() {
-        eprintln!("ignoring careless call to dc_msg_save_file()");
-        return 0;
-    }
-    let ffi_msg = &*msg;
-    let ctx = &*ffi_msg.context;
-    let path = to_string_lossy(path);
-    let r = block_on(
-        ffi_msg
-            .message
-            .save_file(ctx, &std::path::PathBuf::from(path)),
-    );
-    match r {
-        Ok(()) => 1,
-        Err(_) => {
-            r.context("Failed to save file from message")
-                .log_err(ctx)
-                .unwrap_or_default();
-            0
-        }
-    }
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.137.4"
+version = "1.137.3"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -16,11 +16,11 @@ required-features = ["webserver"]
 anyhow = "1"
 deltachat = { path = ".." }
 num-traits = "0.2"
-schemars = "0.8.19"
+schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
 tempfile = "3.10.1"
 log = "0.4"
-async-channel = { version = "2.2.1" }
+async-channel = { version = "2.0.0" }
 futures = { version = "0.3.30" }
 serde_json = "1"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }

deltachat-jsonrpc/src/api.rs

@@ -1,5 +1,4 @@
 use std::collections::BTreeMap;
-use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{collections::HashMap, str::FromStr};
@@ -1771,29 +1770,6 @@ impl CommandApi {
         Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
     }
 
-    /// Sets Webxdc file as integration.
-    /// `file` is the .xdc to use as Webxdc integration.
-    async fn set_webxdc_integration(&self, account_id: u32, file_path: String) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
-        ctx.set_webxdc_integration(&file_path).await
-    }
-
-    /// Returns Webxdc instance used for optional integrations.
-    /// UI can open the Webxdc as usual.
-    /// Returns `None` if there is no integration; the caller can add one using `set_webxdc_integration` then.
-    /// `integrate_for` is the chat to get the integration for.
-    async fn init_webxdc_integration(
-        &self,
-        account_id: u32,
-        chat_id: Option<u32>,
-    ) -> Result<Option<u32>> {
-        let ctx = self.get_context(account_id).await?;
-        Ok(ctx
-            .init_webxdc_integration(chat_id.map(ChatId::new))
-            .await?
-            .map(|msg_id| msg_id.to_u32()))
-    }
-
     /// Makes an HTTP GET request and returns a response.
     ///
     /// `url` is the HTTP or HTTPS URL.
@@ -1902,15 +1878,6 @@ impl CommandApi {
         Ok(can_send)
     }
 
-    /// Saves a file copy at the user-provided path.
-    ///
-    /// Fails if file already exists at the provided path.
-    async fn save_msg_file(&self, account_id: u32, msg_id: u32, path: String) -> Result<()> {
-        let ctx = self.get_context(account_id).await?;
-        let message = Message::load_from_db(&ctx, MsgId::new(msg_id)).await?;
-        message.save_file(&ctx, Path::new(&path)).await
-    }
-
     // ---------------------------------------------
     //           functions for the composer
     //    the composer is the message input field
@@ -1983,21 +1950,19 @@ impl CommandApi {
         );
         let destination_path = account_folder.join("stickers").join(collection);
         fs::create_dir_all(&destination_path).await?;
-        let file = message.get_filename().context("no file?")?;
-        message
-            .save_file(
-                &ctx,
-                &destination_path.join(format!(
-                    "{}.{}",
-                    msg_id,
-                    Path::new(&file)
-                        .extension()
-                        .unwrap_or_default()
-                        .to_str()
-                        .unwrap_or_default()
-                )),
-            )
-            .await?;
+        let file = message.get_file(&ctx).context("no file")?;
+        fs::copy(
+            &file,
+            destination_path.join(format!(
+                "{}.{}",
+                msg_id,
+                file.extension()
+                    .unwrap_or_default()
+                    .to_str()
+                    .unwrap_or_default()
+            )),
+        )
+        .await?;
         Ok(())
     }
 

deltachat-jsonrpc/src/api/types/message.rs

@@ -35,10 +35,6 @@ pub struct MessageObject {
     parent_id: Option<u32>,
 
     text: String,
-
-    /// Check if a message has a POI location bound to it.
-    /// These locations are also returned by `get_locations` method.
-    /// The UI may decide to display a special icon beside such messages.
     has_location: bool,
     has_html: bool,
     view_type: MessageViewtype,

deltachat-jsonrpc/typescript/package.json

@@ -25,8 +25,7 @@
   "exports": {
     ".": {
       "import": "./dist/deltachat.js",
-      "require": "./dist/deltachat.cjs",
-      "types": "./dist/deltachat.d.ts"
+      "require": "./dist/deltachat.cjs"
     }
   },
   "license": "MPL-2.0",
@@ -54,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.137.4"
+  "version": "1.137.3"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.137.4"
+version = "1.137.3"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"

deltachat-rpc-client/examples/echobot.py

@@ -4,7 +4,6 @@
 it will echo back any text send to it, it also will print to console all Delta Chat core events.
 Pass --help to the CLI to see available options.
 """
-
 from deltachat_rpc_client import events, run_bot_cli
 
 hooks = events.HookCollection()

deltachat-rpc-client/examples/echobot_advanced.py

@@ -3,7 +3,6 @@
 
 it will echo back any message that has non-empty text and also supports the /help command.
 """
-
 import logging
 import sys
 from threading import Thread

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -2,7 +2,6 @@
 """
 Example echo bot without using hooks
 """
-
 import logging
 import sys
 

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.137.4"
+version = "1.137.3"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -37,7 +37,7 @@ deltachat_rpc_client = [
 line-length = 120
 
 [tool.ruff]
-lint.select = [
+select = [
   "E", "W", # pycodestyle
   "F", # Pyflakes
   "N", # pep8-naming

deltachat-rpc-client/tox.ini

@@ -22,8 +22,9 @@ skipsdist = True
 skip_install = True
 deps =
     ruff
+    black
 commands =
-    ruff format --quiet --diff src/ examples/ tests/
+    black --quiet --check --diff src/ examples/ tests/
     ruff check src/ examples/ tests/
 
 [pytest]

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.137.4"
+version = "1.137.3"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/.gitignore

@@ -1,2 +0,0 @@
-platform_package
-*.tgz

deltachat-rpc-server/npm-package/.npmignore

@@ -1,3 +0,0 @@
-platform_package/*
-scripts/
-*.tgz

deltachat-rpc-server/npm-package/README.md

@@ -1,77 +0,0 @@
-## npm package for deltachat-rpc-server
-
-This is the successor of `deltachat-node`,
-it does not use NAPI bindings but instead uses stdio executables
-to let you talk to core over jsonrpc over stdio.
-This simplifies cross-compilation and even reduces binary size (no CFFI layer and no NAPI layer).
-
-## Usage
-
-> The **minimum** nodejs version for this package is `20.11`
-
-```
-npm i @deltachat/stdio-rpc-server @deltachat/jsonrpc-client
-```
-
-```js
-import { startDeltaChat } from "@deltachat/stdio-rpc-server";
-import { C } from "@deltachat/jsonrpc-client";
-
-async function main() {
-    const dc = await startDeltaChat("deltachat-data");
-    console.log(await dc.rpc.getSystemInfo());
-}
-```
-
-For a more complete example refer to https://github.com/deltachat-bot/echo/pull/69/files (TODO change link when pr is merged).
-
-## How to use on an unsupported platform
-
-<!-- todo instructions, will uses an env var for pointing to `deltachat-rpc-server` binary -->
-
-<!-- todo copy parts from https://github.com/deltachat/deltachat-desktop/blob/7045c6f549e4b9d5caa0709d5bd314bbd9fd53db/docs/UPDATE_CORE.md -->
-
-## How does it work when you install it
-
-NPM automatically installs platform dependent optional dependencies when `os` and `cpu` fields are set correctly.
-
-references:
-
-- https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages, [webarchive version](https://web.archive.org/web/20240309234250/https://napi.rs/docs/deep-dive/release#3-the-native-addon-for-different-platforms-is-distributed-through-different-npm-packages)
-- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#cpu
-- https://docs.npmjs.com/cli/v6/configuring-npm/package-json#os
-
-When you import this package it searches for the rpc server in the following locations and order:
-
-1. `DELTA_CHAT_RPC_SERVER` environment variable
-2. in PATH
-   - unless `DELTA_CHAT_SKIP_PATH=1` is specified
-   - searches in .cargo/bin directory first
-   - but there an additional version check is performed
-3. prebuilds in npm packages
-
-## How do you built this package in CI
-
-- To build platform packages, run the `build_platform_package.py` script:
-  ```
-  python3 build_platform_package.py <cargo-target>
-  # example
-  python3 build_platform_package.py x86_64-apple-darwin
-  ```
-- Then pass it as an artifact to the last CI action that publishes the main package.
-- upload all packages from `deltachat-rpc-server/npm-package/platform_package`.
-- then publish `deltachat-rpc-server/npm-package`,
-  - this will run `update_optional_dependencies_and_version.js` (in the `prepack` script),
-    which puts all platform packages into `optionalDependencies` and updates the `version` in `package.json`
-
-## How to build a version you can use localy on your host machine for development
-
-You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have seperate scripts for making it work for local installation.
-
-- If you just need your host platform run `python scripts/make_local_dev_version.py`
-- note: this clears the `platform_package` folder
-- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple plaftorms with `build_platform_package.py`
-
-## Thanks to nlnet
-
-The initial work on this package was funded by nlnet as part of the [Delta Tauri](https://nlnet.nl/project/DeltaTauri/) Project.

deltachat-rpc-server/npm-package/index.d.ts

@@ -1,39 +0,0 @@
-import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
-
-export interface SearchOptions {
-  /** whether to disable looking for deltachat-rpc-server inside of $PATH */
-  skipSearchInPath: boolean;
-
-  /** whether to disable the DELTA_CHAT_RPC_SERVER environment variable */
-  disableEnvPath: boolean;
-}
-
-/**
- * 
- * @returns absolute path to deltachat-rpc-server binary
- * @throws when it is not found
- */
-export function getRPCServerPath(
-  options?: Partial<SearchOptions>
-): Promise<string>;
-
-
-
-export type DeltaChatOverJsonRpcServer = StdioDeltaChat & {
-    shutdown: () => Promise<void>;
-    readonly pathToServerBinary: string;
-};
-
-
-/**
- * 
- * @param directory directory for accounts folder
- * @param options 
- */
-export function startDeltaChat(directory: string, options?: Partial<SearchOptions> ): Promise<DeltaChatOverJsonRpcServer>
-
-
-export namespace FnTypes {
-    export type getRPCServerPath = typeof getRPCServerPath
-    export type startDeltaChat = typeof startDeltaChat
-}

deltachat-rpc-server/npm-package/index.js

@@ -1,143 +0,0 @@
-//@ts-check
-import { execFile, spawn } from "node:child_process";
-import { stat, readdir } from "node:fs/promises";
-import os from "node:os";
-import { join, basename } from "node:path";
-import process from "node:process";
-import { promisify } from "node:util";
-import {
-  ENV_VAR_NAME,
-  PATH_EXECUTABLE_NAME,
-  SKIP_SEARCH_IN_PATH,
-} from "./src/const.js";
-import {
-  ENV_VAR_LOCATION_NOT_FOUND,
-  FAILED_TO_START_SERVER_EXECUTABLE,
-  NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR,
-  NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR,
-} from "./src/errors.js";
-
-// Because this is not compiled by typescript, esm needs this stuff (` with { type: "json" };`,
-// nodejs still complains about it being experimental, but deno also uses it, so treefit bets taht it will become standard)
-import package_json from "./package.json" with { type: "json" };
-import { createRequire } from "node:module";
-
-// exports
-// - [ ] a raw starter that has a stdin/out handle thingie like desktop uses
-// - [X] a function that already wraps the stdio handle from above into the deltachat jsonrpc bindings
-
-function findRPCServerInNodeModules() {
-  const arch = os.arch();
-  const operating_system = process.platform;
-  const package_name = `@deltachat/stdio-rpc-server-${operating_system}-${arch}`;
-  try {
-    const { resolve } = createRequire(import.meta.url);
-    return resolve(package_name);
-  } catch (error) {
-    console.debug("findRpcServerInNodeModules", error);
-    if (Object.keys(package_json.optionalDependencies).includes(package_name)) {
-      throw new Error(NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name));
-    } else {
-      throw new Error(NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR());
-    }
-  }
-}
-
-/** @type {import("./index").FnTypes.getRPCServerPath} */
-export async function getRPCServerPath(
-  options = { skipSearchInPath: false, disableEnvPath: false }
-) {
-  // @TODO: improve confusing naming of these options
-  const { skipSearchInPath, disableEnvPath } = options;
-  // 1. check if it is set as env var
-  if (process.env[ENV_VAR_NAME] && !disableEnvPath) {
-    try {
-      if (!(await stat(process.env[ENV_VAR_NAME])).isFile()) {
-        throw new Error(
-          `expected ${ENV_VAR_NAME} to point to the deltachat-rpc-server executable`
-        );
-      }
-    } catch (error) {
-      throw new Error(ENV_VAR_LOCATION_NOT_FOUND());
-    }
-    return process.env[ENV_VAR_NAME];
-  }
-
-  // 2. check if it can be found in PATH
-  if (!process.env[SKIP_SEARCH_IN_PATH] && !skipSearchInPath) {
-    const exec = promisify(execFile);
-
-    const { stdout: executable } =
-      os.platform() !== "win32"
-        ? await exec("command", ["-v", PATH_EXECUTABLE_NAME])
-        : await exec("where", [PATH_EXECUTABLE_NAME]);
-
-    // by just trying to execute it and then use "command -v deltachat-rpc-server" (unix) or "where deltachat-rpc-server" (windows) to get the path to the executable
-    if (executable.length > 1) {
-      // test if it is the right version
-      try {
-        // for some unknown reason it is in stderr and not in stdout
-        const { stderr } = await promisify(execFile)(executable, ["--version"]);
-        const version = stderr.slice(0, stderr.indexOf("\n"));
-        if (package_json.version !== version) {
-          throw new Error(
-            `version mismatch: (npm package: ${package_json.version})  (installed ${PATH_EXECUTABLE_NAME} version: ${version})`
-          );
-        } else {
-          return executable;
-        }
-      } catch (error) {
-        console.error(
-          "Found executable in PATH, but there was an error: " + error
-        );
-        console.error("So falling back to using prebuild...");
-      }
-    }
-  }
-  // 3. check for prebuilds
-
-  return findRPCServerInNodeModules();
-}
-
-import { StdioDeltaChat } from "@deltachat/jsonrpc-client";
-
-/** @type {import("./index").FnTypes.startDeltaChat} */
-export async function startDeltaChat(directory, options) {
-  const pathToServerBinary = await getRPCServerPath(options);
-  const server = spawn(pathToServerBinary, {
-    env: {
-      RUST_LOG: process.env.RUST_LOG || "info",
-      DC_ACCOUNTS_PATH: directory,
-    },
-  });
-
-  server.on("error", (err) => {
-    throw new Error(FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, err));
-  });
-  let shouldClose = false;
-
-  server.on("exit", () => {
-    if (shouldClose) {
-      return;
-    }
-    throw new Error("Server quit");
-  });
-
-  server.stderr.pipe(process.stderr);
-
-  /** @type {import('./index').DeltaChatOverJsonRpcServer} */
-  //@ts-expect-error
-  const dc = new StdioDeltaChat(server.stdin, server.stdout, true);
-
-  dc.shutdown = async () => {
-    shouldClose = true;
-    if (!server.kill()) {
-      console.log("server termination failed");
-    }
-  };
-
-  //@ts-expect-error
-  dc.pathToServerBinary = pathToServerBinary;
-
-  return dc;
-}

deltachat-rpc-server/npm-package/package.json

@@ -1,16 +0,0 @@
-{
-    "type": "module",
-    "name": "@deltachat/stdio-rpc-server",
-    "version": "1.137.4",
-    "license": "MPL-2.0",
-    "main": "index.js",
-    "types": "index.d.ts",
-    "scripts": {
-        "prepack": "node scripts/update_optional_dependencies_and_version.js"
-    },
-    "optionalDependencies": {
-    },
-    "peerDependencies": {
-        "@deltachat/jsonrpc-client": "*"
-    }
-}

deltachat-rpc-server/npm-package/scripts/build_platform_package.py

@@ -1,53 +0,0 @@
-#!/usr/bin/env python3
-import subprocess
-from sys import argv
-from os import path, makedirs, chdir
-from shutil import copy
-from src.make_package import write_package_json
-
-# ensure correct working directory
-chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
-
-if len(argv) < 2:
-    print("First argument should be target architecture as required by cargo")
-    exit(1)
-
-target = argv[1].strip()
-
-subprocess.run(
-    ["cargo", "build", "--release", "-p", "deltachat-rpc-server", "--target", target],
-    check=True,
-)
-
-newpath = "platform_package"
-if not path.exists(newpath):
-    makedirs(newpath)
-
-# make new folder
-
-platform_path = "platform_package/" + target
-if not path.exists(platform_path):
-    makedirs(platform_path)
-
-# copy binary it over
-
-
-def binary_path(binary_name):
-    return "../../target/" + target + "/release/" + binary_name
-
-
-my_binary_name = "deltachat-rpc-server"
-
-if not path.isfile(binary_path("deltachat-rpc-server")):
-    my_binary_name = "deltachat-rpc-server.exe"
-    if not path.isfile(binary_path("deltachat-rpc-server.exe")):
-        print("Did not find the build")
-        exit(1)
-
-my_binary_path = binary_path(my_binary_name)
-
-copy(my_binary_path, platform_path + "/" + my_binary_name)
-
-# make a package.json for it
-
-write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/make_local_dev_version.py

@@ -1,31 +0,0 @@
-# This script is for making a version of the npm packet that you can install locally
-
-import subprocess
-from sys import argv
-from os import path, makedirs, chdir
-import re
-import json
-import tomllib
-from shutil import copy, rmtree
-
-# ensure correct working directory
-chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
-
-# get host target with "rustc -vV"
-output = subprocess.run(["rustc", "-vV"], capture_output=True)
-host_target = re.search('host: ([-\\w]*)', output.stdout.decode("utf-8")).group(1)
-print("host target to build for is:", host_target)
-
-# clean platform_package folder
-newpath = r'platform_package' 
-if not path.exists(newpath):
-    makedirs(newpath)
-else:
-    rmtree(path.join(path.dirname(path.abspath(__file__)), "../platform_package/"))
-    makedirs(newpath)
-
-# run build_platform_package.py with the host's target to build it
-subprocess.run(["python", "scripts/build_platform_package.py", host_target], capture_output=False, check=True)
-
-# run update_optional_dependencies_and_version.js to adjust the package / make it installable locally
-subprocess.run(["node", "scripts/update_optional_dependencies_and_version.js", "--local"], capture_output=False, check=True)

deltachat-rpc-server/npm-package/scripts/pack_binary_for_platform.py

@@ -1,46 +0,0 @@
-import subprocess
-from sys import argv
-from os import path, makedirs, chdir, chmod, stat
-import json
-from shutil import copy
-from src.make_package import write_package_json
-
-# ensure correct working directory
-chdir(path.join(path.dirname(path.abspath(__file__)), "../"))
-
-if len(argv) < 3:
-    print("First argument should be target architecture as required by cargo")
-    print("Second argument should be the location of th built binary (binary_path)")
-    exit(1)
-
-target = argv[1].strip()
-binary_path = argv[2].strip()
-
-output = subprocess.run(["rustc","--print","target-list"], capture_output=True, check=True)
-available_targets = output.stdout.decode("utf-8")
-
-if available_targets.find(target) == -1:
-    print("target", target, "is not known / not valid")
-    exit(1)
-
-
-newpath = r'platform_package' 
-if not path.exists(newpath):
-    makedirs(newpath)
-
-# make new folder
-
-platform_path = 'platform_package/' + target
-if not path.exists(platform_path):
-    makedirs(platform_path)
-
-# copy binary it over
-
-my_binary_name = path.basename(binary_path)
-new_binary_path = platform_path + "/" + my_binary_name
-copy(binary_path, new_binary_path)
-chmod(new_binary_path, 0o555) # everyone can read & execute, nobody can write
-
-# make a package.json for it
-
-write_package_json(platform_path, target, my_binary_name)

deltachat-rpc-server/npm-package/scripts/src/convert_platform.py

@@ -1,21 +0,0 @@
-def convert_cpu_arch_to_npm_cpu_arch(arch):
-    if arch == "x86_64":
-        return "x64"
-    if arch == "i686":
-        return "i32"
-    if arch == "aarch64":
-        return "arm64"
-    if arch == "armv7" or arch == "arm":
-        return "arm"
-    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.arch':", arch)
-    return arch
-
-def convert_os_to_npm_os(os):
-    if os == "windows":
-        return "win32"
-    if os == "darwin" or os == "linux":
-        return os
-    if os.startswith("android"):
-        return "android"
-    print("architecture might not be known by nodejs, please make sure it can be returned by 'process.platform':", os)
-    return os

deltachat-rpc-server/npm-package/scripts/src/make_package.py

@@ -1,27 +0,0 @@
-import tomllib
-import json
-
-from .convert_platform import convert_cpu_arch_to_npm_cpu_arch, convert_os_to_npm_os
-
-def write_package_json(platform_path, rust_target, my_binary_name):
-    if len(rust_target.split("-")) == 3:
-        [cpu_arch, vendor, os] = rust_target.split("-")
-    else:
-        [cpu_arch, vendor, os, _env] = rust_target.split("-")
-
-    # read version
-    tomlfile = open("../../Cargo.toml", 'rb')
-    version = tomllib.load(tomlfile)['package']['version']
-
-    package_json = dict({
-        "name": "@deltachat/stdio-rpc-server-" + convert_os_to_npm_os(os) + "-" + convert_cpu_arch_to_npm_cpu_arch(cpu_arch),
-        "version": version,
-        "os": [convert_os_to_npm_os(os)],
-        "cpu": [convert_cpu_arch_to_npm_cpu_arch(cpu_arch)],
-        "main": my_binary_name,
-        "license": "MPL-2.0"
-    })
-
-    file = open(platform_path + "/package.json", 'w')
-    file.write(json.dumps(package_json, indent=4))
-

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -1,57 +0,0 @@
-import fs from "node:fs/promises";
-import { join, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const expected_cwd = join(dirname(fileURLToPath(import.meta.url)), "..");
-
-if (process.cwd() !== expected_cwd) {
-  console.error(
-    "CWD missmatch: this script needs to be run from " + expected_cwd,
-    { actual: process.cwd(), expected: expected_cwd }
-  );
-  process.exit(1);
-}
-
-// whether to use local paths instead of npm registry version number for the prebuilds in optionalDependencies
-// useful for local development
-const is_local = process.argv.includes("--local");
-
-const package_json = JSON.parse(await fs.readFile("./package.json", "utf8"));
-
-const cargo_toml = await fs.readFile("../Cargo.toml", "utf8");
-const version = cargo_toml
-  .split("\n")
-  .find((line) => line.includes("version"))
-  .split('"')[1];
-
-const platform_packages_dir = "./platform_package";
-
-const platform_package_names = await Promise.all(
-  (await fs.readdir(platform_packages_dir)).map(async (name) => {
-    const p = JSON.parse(
-      await fs.readFile(
-        join(platform_packages_dir, name, "package.json"),
-        "utf8"
-      )
-    );
-    if (p.version !== version) {
-      console.error(
-        name,
-        "has a different version than the version of the rpc server.",
-        { rpc_server: version, platform_package: p.version }
-      );
-      throw new Error("version missmatch");
-    }
-    return { folder_name: name, package_name: p.name };
-  })
-);
-
-package_json.version = version;
-package_json.optionalDependencies = {};
-for (const { folder_name, package_name } of platform_package_names) {
-  package_json.optionalDependencies[package_name] = is_local
-    ? `file:${expected_cwd}/platform_package/${folder_name}` // npm seems to work better with an absolute path here
-    : version;
-}
-
-await fs.writeFile("./package.json", JSON.stringify(package_json, null, 4));

deltachat-rpc-server/npm-package/src/const.js

@@ -1,6 +0,0 @@
-//@ts-check
-
-export const PATH_EXECUTABLE_NAME = 'deltachat-rpc-server'
-
-export const ENV_VAR_NAME = "DELTA_CHAT_RPC_SERVER"
-export const SKIP_SEARCH_IN_PATH = "DELTA_CHAT_SKIP_PATH"

deltachat-rpc-server/npm-package/src/errors.js

@@ -1,41 +0,0 @@
-//@ts-check
-import { ENV_VAR_NAME } from "./const.js";
-
-const cargoInstallCommand =
-  "cargo install --git https://github.com/deltachat/deltachat-core-rust deltachat-rpc-server";
-
-export function NPM_NOT_FOUND_SUPPORTED_PLATFORM_ERROR(package_name) {
-  return `deltachat-rpc-server not found:
-
-- Install it with "npm i ${package_name}"
-- or download/compile deltachat-rpc-server for your platform and
- - either put it into your PATH (for example with "${cargoInstallCommand}")
- - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
-}
-
-export function NPM_NOT_FOUND_UNSUPPORTED_PLATFORM_ERROR() {
-  return `deltachat-rpc-server not found:
-
-Unfortunately no prebuild is available for your system, so you need to provide deltachat-rpc-server yourself.
-
-- Download or Compile deltachat-rpc-server for your platform and
- - either put it into your PATH (for example with "${cargoInstallCommand}")
- - or set the "${ENV_VAR_NAME}" env var to the path to deltachat-rpc-server"`;
-}
-
-export function ENV_VAR_LOCATION_NOT_FOUND(error) {
-  return `deltachat-rpc-server not found in ${ENV_VAR_NAME}:
-
-    Error: ${error}
-
-    Content of ${ENV_VAR_NAME}: "${process.env[ENV_VAR_NAME]}"`;
-}
-
-export function FAILED_TO_START_SERVER_EXECUTABLE(pathToServerBinary, error) {
-  return `Failed to start server executable at '${pathToServerBinary}',
-
-  Error: ${error}
-
-Make sure the deltachat-rpc-server binary exists at this location 
-and you can start it with \`${pathToServerBinary} --version\``;
-}

package.json

@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.137.4"
+  "version": "1.137.3"
 }

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.137.4"
+version = "1.137.3"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.7"
@@ -46,7 +46,7 @@ deltachat = [
 line-length = 120
 
 [tool.ruff]
-lint.select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP004", "UP010", "UP031", "UP032", "ANN204"]
+select = ["E", "F", "W", "YTT", "C4", "ISC", "ICN", "TID", "DTZ", "PLC", "PLE", "PLW", "PIE", "COM", "UP004", "UP010", "UP031", "UP032", "ANN204"]
 line-length = 120
 
 [tool.isort]

python/tests/stress_test_db.py

@@ -18,14 +18,14 @@ def test_db_busy_error(acfactory):
 
     # make a number of accounts
     accounts = acfactory.get_many_online_accounts(3)
-    log(f"created {len(accounts)} accounts")
+    log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account
     for acc in accounts:
         acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
         with open(acc.bigfile, "wb") as f:
             f.write(b"01234567890" * 1000_000)
-    log(f"created {len(accounts)} bigfiles")
+    log("created %s bigfiles" % len(accounts))
 
     contact_addrs = [acc.get_self_contact().addr for acc in accounts]
     chat = accounts[0].create_group_chat("stress-group")

python/tests/test_1_online.py

@@ -1343,6 +1343,7 @@ def test_quote_encrypted(acfactory, lp):
 
     for quoted_msg in msg1, msg3:
         # Save the draft with a quote.
+        # It should be encrypted if quoted message is encrypted.
         msg_draft = Message.new_empty(ac1, "text")
         msg_draft.set_text("message reply")
         msg_draft.quote = quoted_msg
@@ -1356,14 +1357,10 @@ def test_quote_encrypted(acfactory, lp):
         chat.set_draft(None)
         assert chat.get_draft() is None
 
-        # Quote should be replaced with "..." if quoted message is encrypted.
         msg_in = ac2._evtracker.wait_next_incoming_message()
         assert msg_in.text == "message reply"
-        assert not msg_in.is_encrypted()
-        if quoted_msg.is_encrypted():
-            assert msg_in.quoted_text == "..."
-        else:
-            assert msg_in.quoted_text == quoted_msg.text
+        assert msg_in.quoted_text == quoted_msg.text
+        assert msg_in.is_encrypted() == quoted_msg.is_encrypted()
 
 
 def test_quote_attachment(tmp_path, acfactory, lp):
@@ -2036,15 +2033,14 @@ def test_send_receive_locations(acfactory, lp):
     assert chat1.is_sending_locations()
     ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
-    # Wait for "enabled location streaming" message.
-    ac2._evtracker.wait_next_incoming_message()
-
-    # First location is sent immediately as a location-only message.
     ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
     ac1._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
+    chat1.send_text("🍞")
     ac1._evtracker.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
 
     lp.sec("ac2: wait for incoming location message")
+
+    # currently core emits location changed before event_incoming message
     ac2._evtracker.get_matching("DC_EVENT_LOCATION_CHANGED")
 
     locations = chat2.get_locations()
@@ -2053,7 +2049,7 @@ def test_send_receive_locations(acfactory, lp):
     assert locations[0].longitude == 3.0
     assert locations[0].accuracy == 0.5
     assert locations[0].timestamp > now
-    assert locations[0].marker is None
+    assert locations[0].marker == "🍞"
 
     contact = ac2.create_contact(ac1)
     locations2 = chat2.get_locations(contact=contact)

python/tox.ini

@@ -41,11 +41,12 @@ skipsdist = True
 skip_install = True
 deps =
     ruff
+    black
     # pygments required by rst-lint
     pygments
     restructuredtext_lint 
 commands =
-    ruff format --quiet --diff setup.py src/deltachat examples/ tests/
+    black --quiet --check --diff setup.py src/deltachat examples/ tests/
     ruff check src/deltachat tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 

release-date.in

@@ -1 +1 @@
-2024-04-24
+2024-04-16

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.78.0
+RUST_VERSION=1.77.1
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/set_core_version.py

@@ -66,11 +66,7 @@ def main():
     parser = ArgumentParser(prog="set_core_version")
     parser.add_argument("newversion")
 
-    json_list = [
-        "package.json",
-        "deltachat-jsonrpc/typescript/package.json",
-        "deltachat-rpc-server/npm-package/package.json",
-    ]
+    json_list = ["package.json", "deltachat-jsonrpc/typescript/package.json"]
     toml_list = [
         "Cargo.toml",
         "deltachat-ffi/Cargo.toml",

src/authres.rs

@@ -14,6 +14,7 @@ use once_cell::sync::Lazy;
 use crate::config::Config;
 use crate::context::Context;
 use crate::headerdef::HeaderDef;
+use crate::tools::time;
 
 /// `authres` is short for the Authentication-Results header, defined in
 /// <https://datatracker.ietf.org/doc/html/rfc8601>, which contains info
@@ -28,28 +29,45 @@ pub(crate) async fn handle_authres(
     context: &Context,
     mail: &ParsedMail<'_>,
     from: &str,
+    message_time: i64,
 ) -> Result<DkimResults> {
     let from_domain = match EmailAddress::new(from) {
         Ok(email) => email.domain,
         Err(e) => {
+            // This email is invalid, but don't return an error, we still want to
+            // add a stub to the database so that it's not downloaded again
             return Err(anyhow::format_err!("invalid email {}: {:#}", from, e));
         }
     };
 
     let authres = parse_authres_headers(&mail.get_headers(), &from_domain);
     update_authservid_candidates(context, &authres).await?;
-    compute_dkim_results(context, authres).await
+    compute_dkim_results(context, authres, &from_domain, message_time).await
 }
 
 #[derive(Debug)]
 pub(crate) struct DkimResults {
     /// Whether DKIM passed for this particular e-mail.
     pub dkim_passed: bool,
+    /// Whether DKIM is known to work for e-mails coming from the sender's domain,
+    /// i.e. whether we expect DKIM to work.
+    pub dkim_should_work: bool,
+    /// Whether changing the public Autocrypt key should be allowed.
+    /// This is false if we expected DKIM to work (dkim_works=true),
+    /// but it failed now (dkim_passed=false).
+    pub allow_keychange: bool,
 }
 
 impl fmt::Display for DkimResults {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(fmt, "DKIM Results: Passed={}", self.dkim_passed)?;
+        write!(
+            fmt,
+            "DKIM Results: Passed={}, Works={}, Allow_Keychange={}",
+            self.dkim_passed, self.dkim_should_work, self.allow_keychange
+        )?;
+        if !self.allow_keychange {
+            write!(fmt, " KEYCHANGES NOT ALLOWED!!!!")?;
+        }
         Ok(())
     }
 }
@@ -200,6 +218,10 @@ async fn update_authservid_candidates(
         context
             .set_config_internal(Config::AuthservIdCandidates, Some(&new_config))
             .await?;
+        // Updating the authservid candidates may mean that we now consider
+        // emails as "failed" which "passed" previously, so we need to
+        // reset our expectation which DKIMs work.
+        clear_dkim_works(context).await?
     }
     Ok(())
 }
@@ -216,6 +238,8 @@ async fn update_authservid_candidates(
 async fn compute_dkim_results(
     context: &Context,
     mut authres: ParsedAuthresHeaders,
+    from_domain: &str,
+    message_time: i64,
 ) -> Result<DkimResults> {
     let mut dkim_passed = false;
 
@@ -248,7 +272,71 @@ async fn compute_dkim_results(
         }
     }
 
-    Ok(DkimResults { dkim_passed })
+    let last_working_timestamp = dkim_works_timestamp(context, from_domain).await?;
+    let mut dkim_should_work = dkim_should_work(last_working_timestamp)?;
+    if message_time > last_working_timestamp && dkim_passed {
+        set_dkim_works_timestamp(context, from_domain, message_time).await?;
+        dkim_should_work = true;
+    }
+
+    Ok(DkimResults {
+        dkim_passed,
+        dkim_should_work,
+        allow_keychange: dkim_passed || !dkim_should_work,
+    })
+}
+
+/// Whether DKIM in emails from this domain should be considered to work.
+fn dkim_should_work(last_working_timestamp: i64) -> Result<bool> {
+    // When we get an email with valid DKIM-Authentication-Results,
+    // then we assume that DKIM works for 30 days from this time on.
+    let should_work_until = last_working_timestamp + 3600 * 24 * 30;
+
+    let dkim_ever_worked = last_working_timestamp > 0;
+
+    // We're using time() here and not the time when the message
+    // claims to have been sent (passed around as `message_time`)
+    // because otherwise an attacker could just put a time way
+    // in the future into the `Date` header and then we would
+    // assume that DKIM doesn't have to be valid anymore.
+    let dkim_should_work_now = should_work_until > time();
+    Ok(dkim_ever_worked && dkim_should_work_now)
+}
+
+async fn dkim_works_timestamp(context: &Context, from_domain: &str) -> Result<i64, anyhow::Error> {
+    let last_working_timestamp: i64 = context
+        .sql
+        .query_get_value(
+            "SELECT dkim_works FROM sending_domains WHERE domain=?",
+            (from_domain,),
+        )
+        .await?
+        .unwrap_or(0);
+    Ok(last_working_timestamp)
+}
+
+async fn set_dkim_works_timestamp(
+    context: &Context,
+    from_domain: &str,
+    timestamp: i64,
+) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "INSERT INTO sending_domains (domain, dkim_works) VALUES (?,?)
+                ON CONFLICT(domain) DO UPDATE SET dkim_works=excluded.dkim_works",
+            (from_domain, timestamp),
+        )
+        .await?;
+    Ok(())
+}
+
+async fn clear_dkim_works(context: &Context) -> Result<()> {
+    context
+        .sql
+        .execute("DELETE FROM sending_domains", ())
+        .await?;
+    Ok(())
 }
 
 fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str> {
@@ -261,12 +349,19 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 #[cfg(test)]
 mod tests {
     #![allow(clippy::indexing_slicing)]
+    use std::time::Duration;
+
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 
     use super::*;
+    use crate::aheader::EncryptPreference;
+    use crate::e2ee;
     use crate::mimeparser;
     use crate::peerstate::Peerstate;
+    use crate::securejoin::get_securejoin_qr;
+    use crate::securejoin::join_securejoin;
+    use crate::test_utils;
     use crate::test_utils::TestContext;
     use crate::test_utils::TestContextManager;
     use crate::tools;
@@ -479,8 +574,33 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 let mail = mailparse::parse_mail(&bytes)?;
                 let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
 
-                let res = handle_authres(&t, &mail, from).await?;
+                let res = handle_authres(&t, &mail, from, time()).await?;
+                assert!(res.allow_keychange);
+            }
+
+            for entry in &dir {
+                let mut file = fs::File::open(entry.path()).await?;
+                bytes.clear();
+                file.read_to_end(&mut bytes).await.unwrap();
+
+                let mail = mailparse::parse_mail(&bytes)?;
+                let from = &mimeparser::get_from(&mail.headers).unwrap().addr;
+
+                let res = handle_authres(&t, &mail, from, time()).await?;
+                if !res.allow_keychange {
+                    println!(
+                        "!!!!!! FAILURE Receiving {:?}, keychange is not allowed !!!!!!",
+                        entry.path()
+                    );
+                    test_failed = true;
+                }
+
                 let from_domain = EmailAddress::new(from).unwrap().domain;
+                assert_eq!(
+                    res.dkim_should_work,
+                    dkim_should_work(dkim_works_timestamp(&t, &from_domain).await?)?
+                );
+                assert_eq!(res.dkim_passed, res.dkim_should_work);
 
                 // delta.blinzeln.de and gmx.de have invalid DKIM, so the DKIM check should fail
                 let expected_result = (from_domain != "delta.blinzeln.de") && (from_domain != "gmx.de")
@@ -493,8 +613,9 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
                 if res.dkim_passed != expected_result {
                     if authres_parsing_works {
                         println!(
-                            "!!!!!! FAILURE Receiving {:?} wrong result: !!!!!!",
+                            "!!!!!! FAILURE Receiving {:?}, order {:#?} wrong result: !!!!!!",
                             entry.path(),
+                            dir.iter().map(|e| e.file_name()).collect::<Vec<_>>()
                         );
                         test_failed = true;
                     }
@@ -517,7 +638,116 @@ Authentication-Results: box.hispanilandia.net; spf=pass smtp.mailfrom=adbenitez@
         let bytes = b"From: invalid@from.com
 Authentication-Results: dkim=";
         let mail = mailparse::parse_mail(bytes).unwrap();
-        handle_authres(&t, &mail, "invalid@rom.com").await.unwrap();
+        handle_authres(&t, &mail, "invalid@rom.com", time())
+            .await
+            .unwrap();
+    }
+
+    #[ignore = "Disallowing keychanges is disabled for now"]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_handle_authres_fails() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        // Bob sends Alice a message, so she gets his key
+        tcm.send_recv_accept(&bob, &alice, "Hi").await;
+
+        // We don't need bob anymore, let's make sure it's not accidentally used
+        drop(bob);
+
+        // Assume Alice receives an email from bob@example.net with
+        // correct DKIM -> `set_dkim_works()` was called
+        set_dkim_works_timestamp(&alice, "example.net", time()).await?;
+        // And Alice knows her server's authserv-id
+        alice
+            .set_config(Config::AuthservIdCandidates, Some("example.org"))
+            .await?;
+
+        tcm.section("An attacker, bob2, sends a from-forged email to Alice!");
+
+        // Sleep to make sure key reset is ignored because of DKIM failure
+        // and not because reordering is suspected.
+        tokio::time::sleep(std::time::Duration::from_millis(1100)).await;
+
+        let bob2 = tcm.unconfigured().await;
+        bob2.configure_addr("bob@example.net").await;
+        e2ee::ensure_secret_key_exists(&bob2).await?;
+
+        let chat = bob2.create_chat(&alice).await;
+        let mut sent = bob2
+            .send_text(chat.id, "Please send me lots of money")
+            .await;
+
+        sent.payload
+            .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
+
+        let received = alice.recv_msg(&sent).await;
+
+        // Assert that the error tells the user about the problem
+        assert!(received.error.unwrap().contains("DKIM failed"));
+
+        let bob_state = Peerstate::from_addr(&alice, "bob@example.net")
+            .await?
+            .unwrap();
+
+        // Encryption preference is still mutual.
+        assert_eq!(bob_state.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Also check that the keypair was not changed
+        assert_eq!(
+            bob_state.public_key.unwrap(),
+            test_utils::bob_keypair().public
+        );
+
+        // Since Alice didn't change the key, Bob can't read her message
+        let received = tcm
+            .try_send_recv(&alice, &bob2, "My credit card number is 1234")
+            .await;
+        assert!(!received.text.contains("1234"));
+        assert!(received.error.is_some());
+
+        tcm.section("Turns out bob2 wasn't an attacker at all, Bob just has a new phone and DKIM just stopped working.");
+        tcm.section("To fix the key problems, Bob scans Alice's QR code.");
+
+        let qr = get_securejoin_qr(&alice.ctx, None).await.unwrap();
+        join_securejoin(&bob2.ctx, &qr).await.unwrap();
+
+        loop {
+            if let Some(mut sent) = bob2.pop_sent_msg_opt(Duration::ZERO).await {
+                sent.payload
+                    .insert_str(0, "Authentication-Results: example.org; dkim=fail\n");
+                alice.recv_msg(&sent).await;
+            } else if let Some(sent) = alice.pop_sent_msg_opt(Duration::ZERO).await {
+                bob2.recv_msg(&sent).await;
+            } else {
+                break;
+            }
+        }
+
+        // Unfortunately, securejoin currently doesn't work with authres-checking,
+        // so these checks would fail:
+
+        // let contact_bob = alice.add_or_lookup_contact(&bob2).await;
+        // assert_eq!(
+        //     contact_bob.is_verified(&alice.ctx).await.unwrap(),
+        //     VerifiedStatus::BidirectVerified
+        // );
+
+        // let contact_alice = bob2.add_or_lookup_contact(&alice).await;
+        // assert_eq!(
+        //     contact_alice.is_verified(&bob2.ctx).await.unwrap(),
+        //     VerifiedStatus::BidirectVerified
+        // );
+
+        // // Bob can read Alice's messages again
+        // let received = tcm
+        //     .try_send_recv(&alice, &bob2, "Can you read this again?")
+        //     .await;
+        // assert_eq!(received.text.as_ref().unwrap(), "Can you read this again?");
+        // assert!(received.error.is_none());
+
+        Ok(())
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -566,7 +796,10 @@ Authentication-Results: dkim=";
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
 
-        // Bob knows his server's authserv-id
+        // Assume Bob received an email from something@example.net with
+        // correct DKIM -> `set_dkim_works()` was called
+        set_dkim_works_timestamp(&bob, "example.org", time()).await?;
+        // And Bob knows his server's authserv-id
         bob.set_config(Config::AuthservIdCandidates, Some("example.net"))
             .await?;
 
@@ -588,13 +821,15 @@ Authentication-Results: dkim=";
             .insert_str(0, "Authentication-Results: example.net; dkim=fail\n");
         let rcvd = bob.recv_msg(&sent).await;
 
+        // Disallowing keychanges is disabled for now:
+        // assert!(rcvd.error.unwrap().contains("DKIM failed"));
         // The message info should contain a warning:
         assert!(rcvd
             .id
             .get_info(&bob)
             .await
             .unwrap()
-            .contains("DKIM Results: Passed=false"));
+            .contains("KEYCHANGES NOT ALLOWED"));
 
         Ok(())
     }

src/blob.rs

@@ -698,10 +698,7 @@ fn encode_img(
         ImageOutputFormat::Png => img.write_to(&mut buf, ImageFormat::Png)?,
         ImageOutputFormat::Jpeg { quality } => {
             let encoder = JpegEncoder::new_with_quality(&mut buf, quality);
-            // Convert image into RGB8 to avoid the error
-            // "The encoder or decoder for Jpeg does not support the color type Rgba8"
-            // (<https://github.com/image-rs/image/issues/2211>).
-            img.clone().into_rgb8().write_with_encoder(encoder)?;
+            img.write_with_encoder(encoder)?;
         }
     }
     Ok(())
@@ -1208,28 +1205,6 @@ mod tests {
         .unwrap();
     }
 
-    /// Tests that RGBA PNG can be recoded into JPEG
-    /// by dropping alpha channel.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_recode_image_rgba_png_to_jpeg() {
-        let bytes = include_bytes!("../test-data/image/screenshot-rgba.png");
-
-        send_image_check_mediaquality(
-            Viewtype::Image,
-            Some("1"),
-            bytes,
-            "png",
-            false, // no Exif
-            1920,
-            1080,
-            0,
-            constants::WORSE_IMAGE_SIZE,
-            constants::WORSE_IMAGE_SIZE * 1080 / 1920,
-        )
-        .await
-        .unwrap();
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_recode_image_huge_jpg() {
         let bytes = include_bytes!("../test-data/image/screenshot.jpg");
@@ -1307,65 +1282,26 @@ mod tests {
         let alice_msg = alice.get_last_msg().await;
         assert_eq!(alice_msg.get_width() as u32, compressed_width);
         assert_eq!(alice_msg.get_height() as u32, compressed_height);
-        let file_saved = alice
-            .get_blobdir()
-            .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
-        alice_msg.save_file(&alice, &file_saved).await?;
-        check_image_size(file_saved, compressed_width, compressed_height);
+        check_image_size(
+            alice_msg.get_file(&alice).unwrap(),
+            compressed_width,
+            compressed_height,
+        );
 
         let bob_msg = bob.recv_msg(&sent).await;
         assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
         assert_eq!(bob_msg.get_width() as u32, compressed_width);
         assert_eq!(bob_msg.get_height() as u32, compressed_height);
-        let file_saved = bob
-            .get_blobdir()
-            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
-        bob_msg.save_file(&bob, &file_saved).await?;
+        let file = bob_msg.get_file(&bob).unwrap();
 
-        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
+        let blob = BlobObject::new_from_path(&bob, &file).await?;
         let (_, exif) = blob.metadata()?;
         assert!(exif.is_none());
 
-        let img = check_image_size(file_saved, compressed_width, compressed_height);
+        let img = check_image_size(file, compressed_width, compressed_height);
         Ok(img)
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_send_big_gif_as_image() -> Result<()> {
-        let bytes = include_bytes!("../test-data/image/screenshot.gif");
-        let (width, height) = (1920u32, 1080u32);
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-        alice
-            .set_config(
-                Config::MediaQuality,
-                Some(&(MediaQuality::Worse as i32).to_string()),
-            )
-            .await?;
-        let file = alice.get_blobdir().join("file").with_extension("gif");
-        fs::write(&file, &bytes)
-            .await
-            .context("failed to write file")?;
-        let mut msg = Message::new(Viewtype::Image);
-        msg.set_file(file.to_str().unwrap(), None);
-        let chat = alice.create_chat(&bob).await;
-        let sent = alice.send_msg(chat.id, &mut msg).await;
-        let bob_msg = bob.recv_msg(&sent).await;
-        // DC must detect the image as GIF and send it w/o reencoding.
-        assert_eq!(bob_msg.get_viewtype(), Viewtype::Gif);
-        assert_eq!(bob_msg.get_width() as u32, width);
-        assert_eq!(bob_msg.get_height() as u32, height);
-        let file_saved = bob
-            .get_blobdir()
-            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
-        bob_msg.save_file(&bob, &file_saved).await?;
-        let blob = BlobObject::new_from_path(&bob, &file_saved).await?;
-        let (file_size, _) = blob.metadata()?;
-        assert_eq!(file_size, bytes.len() as u64);
-        check_image_size(file_saved, width, height);
-        Ok(())
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_increation_in_blobdir() -> Result<()> {
         let t = TestContext::new_alice().await;

src/chat.rs

@@ -2235,9 +2235,8 @@ pub struct ChatInfo {
 }
 
 pub(crate) async fn update_saved_messages_icon(context: &Context) -> Result<()> {
-    if let Some(ChatIdBlocked { id: chat_id, .. }) =
-        ChatIdBlocked::lookup_by_contact(context, ContactId::SELF).await?
-    {
+    // if there is no saved-messages chat, there is nothing to update. this is no error.
+    if let Some(chat_id) = ChatId::lookup_by_contact(context, ContactId::SELF).await? {
         let icon = include_bytes!("../assets/icon-saved-messages.png");
         let blob = BlobObject::create(context, "icon-saved-messages.png", icon).await?;
         let icon = blob.as_name().to_string();
@@ -2250,9 +2249,8 @@ pub(crate) async fn update_saved_messages_icon(context: &Context) -> Result<()>
 }
 
 pub(crate) async fn update_device_icon(context: &Context) -> Result<()> {
-    if let Some(ChatIdBlocked { id: chat_id, .. }) =
-        ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE).await?
-    {
+    // if there is no device-chat, there is nothing to update. this is no error.
+    if let Some(chat_id) = ChatId::lookup_by_contact(context, ContactId::DEVICE).await? {
         let icon = include_bytes!("../assets/icon-device.png");
         let blob = BlobObject::create(context, "icon-device.png", icon).await?;
         let icon = blob.as_name().to_string();
@@ -2303,9 +2301,7 @@ async fn update_special_chat_name(
     contact_id: ContactId,
     name: String,
 ) -> Result<()> {
-    if let Some(ChatIdBlocked { id: chat_id, .. }) =
-        ChatIdBlocked::lookup_by_contact(context, contact_id).await?
-    {
+    if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
         // the `!= name` condition avoids unneeded writes
         context
             .sql
@@ -2509,30 +2505,6 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
             .await?
             .with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?;
 
-        if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
-            // Correct the type, take care not to correct already very special
-            // formats as GIF or VOICE.
-            //
-            // Typical conversions:
-            // - from FILE to AUDIO/VIDEO/IMAGE
-            // - from FILE/IMAGE to GIF */
-            if let Some((better_type, _)) = message::guess_msgtype_from_suffix(&blob.to_abs_path())
-            {
-                if better_type != Viewtype::Webxdc
-                    || context
-                        .ensure_sendable_webxdc_file(&blob.to_abs_path())
-                        .await
-                        .is_ok()
-                {
-                    msg.viewtype = better_type;
-                }
-            }
-        } else if msg.viewtype == Viewtype::Webxdc {
-            context
-                .ensure_sendable_webxdc_file(&blob.to_abs_path())
-                .await?;
-        }
-
         let mut maybe_sticker = msg.viewtype == Viewtype::Sticker;
         if msg.viewtype == Viewtype::Image
             || maybe_sticker && !msg.param.exists(Param::ForceSticker)
@@ -2554,6 +2526,34 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
                 .set(Param::Filename, stem.to_string() + "." + blob_ext);
         }
 
+        if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
+            // Correct the type, take care not to correct already very special
+            // formats as GIF or VOICE.
+            //
+            // Typical conversions:
+            // - from FILE to AUDIO/VIDEO/IMAGE
+            // - from FILE/IMAGE to GIF */
+            if let Some((better_type, better_mime)) =
+                message::guess_msgtype_from_suffix(&blob.to_abs_path())
+            {
+                if better_type != Viewtype::Webxdc
+                    || context
+                        .ensure_sendable_webxdc_file(&blob.to_abs_path())
+                        .await
+                        .is_ok()
+                {
+                    msg.viewtype = better_type;
+                    if !msg.param.exists(Param::MimeType) {
+                        msg.param.set(Param::MimeType, better_mime);
+                    }
+                }
+            }
+        } else if msg.viewtype == Viewtype::Webxdc {
+            context
+                .ensure_sendable_webxdc_file(&blob.to_abs_path())
+                .await?;
+        }
+
         if !msg.param.exists(Param::MimeType) {
             if let Some((_, mime)) = message::guess_msgtype_from_suffix(&blob.to_abs_path()) {
                 msg.param.set(Param::MimeType, mime);
@@ -2597,18 +2597,6 @@ async fn prepare_msg_common(
         }
     }
 
-    // Check a quote reply is not leaking data from other chats.
-    // This is meant as a last line of defence, the UI should check that before as well.
-    // (We allow Chattype::Single in general for "Reply Privately";
-    // checking for exact contact_id will produce false positives when ppl just left the group)
-    if chat.typ != Chattype::Single && !context.get_config_bool(Config::Bot).await? {
-        if let Some(quoted_message) = msg.quoted_message(context).await? {
-            if quoted_message.chat_id != chat_id {
-                bail!("Bad quote reply");
-            }
-        }
-    }
-
     // check current MessageState for drafts (to keep msg_id) ...
     let update_msg_id = if msg.state == MessageState::OutDraft {
         msg.hidden = false;
@@ -2851,10 +2839,17 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
             .await?;
     }
 
-    if rendered_msg.last_added_location_id.is_some() {
+    if let Some(last_added_location_id) = rendered_msg.last_added_location_id {
         if let Err(err) = location::set_kml_sent_timestamp(context, msg.chat_id, now).await {
             error!(context, "Failed to set kml sent_timestamp: {err:#}.");
         }
+        if !msg.hidden {
+            if let Err(err) =
+                location::set_msg_location_id(context, msg.id, last_added_location_id).await
+            {
+                error!(context, "Failed to set msg_location_id: {err:#}.");
+            }
+        }
     }
 
     if let Some(sync_ids) = rendered_msg.sync_ids_to_delete {
@@ -4480,10 +4475,9 @@ impl Context {
                     }
                     _ => (),
                 }
-                ChatIdBlocked::lookup_by_contact(self, contact_id)
+                ChatId::lookup_by_contact(self, contact_id)
                     .await?
                     .with_context(|| format!("No chat for addr '{addr}'"))?
-                    .id
             }
             SyncId::Grpid(grpid) => {
                 if let SyncAction::CreateBroadcast(name) = action {
@@ -4730,59 +4724,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_quote_replies() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let grp_chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "grp").await?;
-        let grp_msg_id = send_text_msg(&alice, grp_chat_id, "bar".to_string()).await?;
-        let grp_msg = Message::load_from_db(&alice, grp_msg_id).await?;
-
-        let one2one_chat_id = alice.create_chat(&bob).await.id;
-        let one2one_msg_id = send_text_msg(&alice, one2one_chat_id, "foo".to_string()).await?;
-        let one2one_msg = Message::load_from_db(&alice, one2one_msg_id).await?;
-
-        // quoting messages in same chat is okay
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("baz".to_string());
-        msg.set_quote(&alice, Some(&grp_msg)).await?;
-        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
-        assert!(result.is_ok());
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("baz".to_string());
-        msg.set_quote(&alice, Some(&one2one_msg)).await?;
-        let result = send_msg(&alice, one2one_chat_id, &mut msg).await;
-        assert!(result.is_ok());
-        let one2one_quote_reply_msg_id = result.unwrap();
-
-        // quoting messages from groups to one-to-ones is okay ("reply privately")
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("baz".to_string());
-        msg.set_quote(&alice, Some(&grp_msg)).await?;
-        let result = send_msg(&alice, one2one_chat_id, &mut msg).await;
-        assert!(result.is_ok());
-
-        // quoting messages from one-to-one chats in groups is an error; usually this is also not allowed by UI at all ...
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text("baz".to_string());
-        msg.set_quote(&alice, Some(&one2one_msg)).await?;
-        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
-        assert!(result.is_err());
-
-        // ... but forwarding messages with quotes is allowed
-        let result = forward_msgs(&alice, &[one2one_quote_reply_msg_id], grp_chat_id).await;
-        assert!(result.is_ok());
-
-        // ... and bots are not restricted
-        alice.set_config(Config::Bot, Some("1")).await?;
-        let result = send_msg(&alice, grp_chat_id, &mut msg).await;
-        assert!(result.is_ok());
-
-        Ok(())
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_add_contact_to_chat_ex_add_self() {
         // Adding self to a contact should succeed, even though it's pointless.
@@ -5097,32 +5038,6 @@ mod tests {
         Ok(())
     }
 
-    /// Tests that if member added message is completely lost,
-    /// member is eventually added.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_lost_member_added() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-        let alice_chat_id = alice
-            .create_group_with_members(ProtectionStatus::Unprotected, "Group", &[bob])
-            .await;
-        let alice_sent = alice.send_text(alice_chat_id, "Hi!").await;
-        let bob_chat_id = bob.recv_msg(&alice_sent).await.chat_id;
-        assert_eq!(get_chat_contacts(bob, bob_chat_id).await?.len(), 2);
-
-        // Attempt to add member, but message is lost.
-        let claire_id = Contact::create(alice, "", "claire@foo.de").await?;
-        add_contact_to_chat(alice, alice_chat_id, claire_id).await?;
-        alice.pop_sent_msg().await;
-
-        let alice_sent = alice.send_text(alice_chat_id, "Hi again!").await;
-        bob.recv_msg(&alice_sent).await;
-        assert_eq!(get_chat_contacts(bob, bob_chat_id).await?.len(), 3);
-
-        Ok(())
-    }
-
     /// Test that group updates are robust to lost messages and eventual out of order arrival.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_modify_chat_lost() -> Result<()> {

src/config.rs

@@ -1007,15 +1007,6 @@ mod tests {
             .set_config(Config::Selfavatar, Some(file.to_str().unwrap()))
             .await?;
         sync(&alice0, &alice1).await;
-        // There was a bug that a sync message creates the self-chat with the user avatar instead of
-        // the special icon and that remains so when the self-chat becomes user-visible. Let's check
-        // this.
-        let self_chat = alice0.get_self_chat().await;
-        let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
-        assert_eq!(
-            self_chat_avatar_path,
-            alice0.get_blobdir().join("icon-saved-messages.png")
-        );
         assert!(alice1
             .get_config(Config::Selfavatar)
             .await?

src/contact.rs

@@ -20,7 +20,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{ChatId, ChatIdBlocked, ProtectionStatus};
+use crate::chat::{ChatId, ProtectionStatus};
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -1315,9 +1315,7 @@ impl Contact {
     pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
         let contact_id = self.id;
 
-        if let Some(ChatIdBlocked { id: chat_id, .. }) =
-            ChatIdBlocked::lookup_by_contact(context, contact_id).await?
-        {
+        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
             Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
         } else {
             // 1:1 chat does not exist.
@@ -2710,19 +2708,7 @@ Hi."#;
 
         bob.recv_msg(&sent_msg).await;
         let contact = Contact::get_by_id(&bob, *contacts.first().unwrap()).await?;
-
-        let green = ansi_term::Color::Green.normal();
-        assert!(
-            contact.was_seen_recently(),
-            "{}",
-            green.paint(
-                "\nNOTE: This test failure is probably a false-positive, caused by tests running in parallel.
-The issue is that `SystemTime::shift()` (a utility function for tests) changes the time for all threads doing tests, and not only for the running test.
-Until the false-positive is fixed:
-- Use `cargo test -- --test-threads 1` instead of `cargo test`
-- Or use `cargo nextest run` (install with `cargo install cargo-nextest --locked`)\n"
-            )
-        );
+        assert!(contact.was_seen_recently());
 
         let self_contact = Contact::get_by_id(&bob, ContactId::SELF).await?;
         assert!(!self_contact.was_seen_recently());

src/decrypt.rs

@@ -57,7 +57,11 @@ pub(crate) async fn prepare_decryption(
             autocrypt_header: None,
             peerstate: None,
             message_time,
-            dkim_results: DkimResults { dkim_passed: false },
+            dkim_results: DkimResults {
+                dkim_passed: false,
+                dkim_should_work: false,
+                allow_keychange: true,
+            },
         });
     }
 
@@ -82,13 +86,15 @@ pub(crate) async fn prepare_decryption(
         None
     };
 
-    let dkim_results = handle_authres(context, mail, from).await?;
+    let dkim_results = handle_authres(context, mail, from, message_time).await?;
     let allow_aeap = get_encrypted_mime(mail).is_some();
     let peerstate = get_autocrypt_peerstate(
         context,
         from,
         autocrypt_header.as_ref(),
         message_time,
+        // Disallowing keychanges is disabled for now:
+        true, // dkim_results.allow_keychange,
         allow_aeap,
     )
     .await?;
@@ -281,15 +287,19 @@ pub(crate) fn keyring_from_peerstate(peerstate: Option<&Peerstate>) -> Vec<Signe
 /// If we already know this fingerprint from another contact's peerstate, return that
 /// peerstate in order to make AEAP work, but don't save it into the db yet.
 ///
+/// The param `allow_change` is used to prevent the autocrypt key from being changed
+/// if we suspect that the message may be forged and have a spoofed sender identity.
+///
 /// Returns updated peerstate.
 pub(crate) async fn get_autocrypt_peerstate(
     context: &Context,
     from: &str,
     autocrypt_header: Option<&Aheader>,
     message_time: i64,
+    allow_change: bool,
     allow_aeap: bool,
 ) -> Result<Option<Peerstate>> {
-    let allow_change = !context.is_self_addr(from).await?;
+    let allow_change = allow_change && !context.is_self_addr(from).await?;
     let mut peerstate;
 
     // Apply Autocrypt header

src/ephemeral.rs

@@ -74,13 +74,12 @@ use async_channel::Receiver;
 use serde::{Deserialize, Serialize};
 use tokio::time::timeout;
 
-use crate::chat::{send_msg, ChatId, ChatIdBlocked};
+use crate::chat::{send_msg, ChatId};
 use crate::constants::{DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH};
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
-use crate::location;
 use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
@@ -349,16 +348,16 @@ pub(crate) async fn start_ephemeral_timers_msgids(
 /// Selects messages which are expired according to
 /// `delete_device_after` setting or `ephemeral_timestamp` column.
 ///
-/// For each message a row ID, chat id, viewtype and location ID is returned.
+/// For each message a row ID, chat id and viewtype is returned.
 async fn select_expired_messages(
     context: &Context,
     now: i64,
-) -> Result<Vec<(MsgId, ChatId, Viewtype, u32)>> {
+) -> Result<Vec<(MsgId, ChatId, Viewtype)>> {
     let mut rows = context
         .sql
         .query_map(
             r#"
-SELECT id, chat_id, type, location_id
+SELECT id, chat_id, type
 FROM msgs
 WHERE
   ephemeral_timestamp != 0
@@ -370,21 +369,18 @@ WHERE
                 let id: MsgId = row.get("id")?;
                 let chat_id: ChatId = row.get("chat_id")?;
                 let viewtype: Viewtype = row.get("type")?;
-                let location_id: u32 = row.get("location_id")?;
-                Ok((id, chat_id, viewtype, location_id))
+                Ok((id, chat_id, viewtype))
             },
             |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
         )
         .await?;
 
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
             .await?
-            .map(|c| c.id)
             .unwrap_or_default();
-        let device_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
             .await?
-            .map(|c| c.id)
             .unwrap_or_default();
 
         let threshold_timestamp = now.saturating_sub(delete_device_after);
@@ -393,7 +389,7 @@ WHERE
             .sql
             .query_map(
                 r#"
-SELECT id, chat_id, type, location_id
+SELECT id, chat_id, type
 FROM msgs
 WHERE
   timestamp < ?1
@@ -412,8 +408,7 @@ WHERE
                     let id: MsgId = row.get("id")?;
                     let chat_id: ChatId = row.get("chat_id")?;
                     let viewtype: Viewtype = row.get("type")?;
-                    let location_id: u32 = row.get("location_id")?;
-                    Ok((id, chat_id, viewtype, location_id))
+                    Ok((id, chat_id, viewtype))
                 },
                 |rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
             )
@@ -444,7 +439,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
 
                 // If you change which information is removed here, also change MsgId::trash() and
                 // which information receive_imf::add_parts() still adds to the db if the chat_id is TRASH
-                for (msg_id, chat_id, viewtype, location_id) in rows {
+                for (msg_id, chat_id, viewtype) in rows {
                     transaction.execute(
                         "UPDATE msgs
                      SET chat_id=?, txt='', subject='', txt_raw='',
@@ -453,13 +448,6 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
                         (DC_CHAT_ID_TRASH, msg_id),
                     )?;
 
-                    if location_id > 0 {
-                        transaction.execute(
-                            "DELETE FROM locations WHERE independent=1 AND id=?",
-                            (location_id,),
-                        )?;
-                    }
-
                     msgs_changed.push((chat_id, msg_id));
                     if viewtype == Viewtype::Webxdc {
                         webxdc_deleted.push(msg_id)
@@ -492,13 +480,11 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
 /// `delete_device_after` setting being set.
 async fn next_delete_device_after_timestamp(context: &Context) -> Result<Option<i64>> {
     if let Some(delete_device_after) = context.get_config_delete_device_after().await? {
-        let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
+        let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
             .await?
-            .map(|c| c.id)
             .unwrap_or_default();
-        let device_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE)
+        let device_chat_id = ChatId::lookup_by_contact(context, ContactId::DEVICE)
             .await?
-            .map(|c| c.id)
             .unwrap_or_default();
 
         let oldest_message_timestamp: Option<i64> = context
@@ -606,11 +592,6 @@ pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiv
             .await
             .log_err(context)
             .ok();
-
-        location::delete_expired(context, time())
-            .await
-            .log_err(context)
-            .ok();
     }
 }
 
@@ -690,10 +671,8 @@ mod tests {
     use super::*;
     use crate::config::Config;
     use crate::download::DownloadState;
-    use crate::location;
-    use crate::message::markseen_msgs;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::{TestContext, TestContextManager};
+    use crate::test_utils::TestContext;
     use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE;
     use crate::{
         chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
@@ -1370,44 +1349,4 @@ mod tests {
 
         Ok(())
     }
-
-    /// Tests that POI location is deleted when ephemeral message expires.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_poi_location() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-
-        let chat = alice.create_chat(bob).await;
-
-        let duration = 60;
-        chat.id
-            .set_ephemeral_timer(alice, Timer::Enabled { duration })
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-
-        let mut poi_msg = Message::new(Viewtype::Text);
-        poi_msg.text = "Here".to_string();
-        poi_msg.set_location(10.0, 20.0);
-
-        let alice_sent_message = alice.send_msg(chat.id, &mut poi_msg).await;
-        let bob_received_message = bob.recv_msg(&alice_sent_message).await;
-        markseen_msgs(bob, vec![bob_received_message.id]).await?;
-
-        for account in [alice, bob] {
-            let locations = location::get_range(account, None, None, 0, 0).await?;
-            assert_eq!(locations.len(), 1);
-        }
-
-        SystemTime::shift(Duration::from_secs(100));
-
-        for account in [alice, bob] {
-            delete_expired_messages(account, time()).await?;
-            let locations = location::get_range(account, None, None, 0, 0).await?;
-            assert_eq!(locations.len(), 0);
-        }
-
-        Ok(())
-    }
 }

src/imap.rs

@@ -5,12 +5,11 @@
 
 use std::{
     cmp::max,
-    cmp::min,
     collections::{BTreeMap, BTreeSet, HashMap},
     iter::Peekable,
     mem::take,
     sync::atomic::Ordering,
-    time::{Duration, UNIX_EPOCH},
+    time::Duration,
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
@@ -20,8 +19,8 @@ use deltachat_contact_tools::{normalize_name, ContactAddress};
 use futures::{FutureExt as _, StreamExt, TryStreamExt};
 use futures_lite::FutureExt;
 use num_traits::FromPrimitive;
-use rand::Rng;
 use ratelimit::Ratelimit;
+use tokio::sync::RwLock;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::chatlist_events;
@@ -43,7 +42,7 @@ use crate::scheduler::connectivity::ConnectivityStore;
 use crate::socks::Socks5Config;
 use crate::sql;
 use crate::stock_str;
-use crate::tools::{self, create_id, duration_to_str};
+use crate::tools::{create_id, duration_to_str};
 
 pub(crate) mod capabilities;
 mod client;
@@ -83,17 +82,15 @@ pub(crate) struct Imap {
 
     pub(crate) connectivity: ConnectivityStore,
 
-    conn_last_try: tools::Time,
-    conn_backoff_ms: u64,
-
-    /// Rate limit for successful IMAP connections.
+    /// Rate limit for IMAP connection attempts.
     ///
-    /// This rate limit prevents busy loop in case the server refuses logins
+    /// This rate limit prevents busy loop
+    /// in case the server refuses connections
     /// or in case connection gets dropped over and over due to IMAP bug,
     /// e.g. the server returning invalid response to SELECT command
     /// immediately after logging in or returning an error in response to LOGIN command
     /// due to internal server error.
-    ratelimit: Ratelimit,
+    ratelimit: RwLock<Ratelimit>,
 }
 
 #[derive(Debug)]
@@ -251,10 +248,8 @@ impl Imap {
             strict_tls,
             login_failed_once: false,
             connectivity: Default::default(),
-            conn_last_try: UNIX_EPOCH,
-            conn_backoff_ms: 0,
             // 1 connection per minute + a burst of 2.
-            ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
+            ratelimit: RwLock::new(Ratelimit::new(Duration::new(120, 0), 2.0)),
         };
 
         Ok(imap)
@@ -298,15 +293,7 @@ impl Imap {
             bail!("IMAP operation attempted while it is torn down");
         }
 
-        let now = tools::Time::now();
-        let until_can_send = max(
-            min(self.conn_last_try, now)
-                .checked_add(Duration::from_millis(self.conn_backoff_ms))
-                .unwrap_or(now),
-            now,
-        )
-        .duration_since(now)?;
-        let ratelimit_duration = max(until_can_send, self.ratelimit.until_can_send());
+        let ratelimit_duration = self.ratelimit.read().await.until_can_send();
         if !ratelimit_duration.is_zero() {
             warn!(
                 context,
@@ -329,16 +316,7 @@ impl Imap {
 
         info!(context, "Connecting to IMAP server");
         self.connectivity.set_connecting(context).await;
-
-        self.conn_last_try = tools::Time::now();
-        const BACKOFF_MIN_MS: u64 = 2000;
-        const BACKOFF_MAX_MS: u64 = 80_000;
-        self.conn_backoff_ms = min(self.conn_backoff_ms, BACKOFF_MAX_MS / 2);
-        self.conn_backoff_ms = self.conn_backoff_ms.saturating_add(
-            rand::thread_rng().gen_range((self.conn_backoff_ms / 2)..=self.conn_backoff_ms),
-        );
-        self.conn_backoff_ms = max(BACKOFF_MIN_MS, self.conn_backoff_ms);
-
+        self.ratelimit.write().await.send();
         let connection_res: Result<Client> =
             if self.lp.security == Socket::Starttls || self.lp.security == Socket::Plain {
                 let imap_server: &str = self.lp.server.as_ref();
@@ -386,8 +364,6 @@ impl Imap {
                 }
             };
         let client = connection_res?;
-        self.conn_backoff_ms = BACKOFF_MIN_MS;
-        self.ratelimit.send();
 
         let imap_user: &str = self.lp.user.as_ref();
         let imap_pw: &str = self.lp.password.as_ref();

src/imex.rs

@@ -193,9 +193,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes())
-        .await?
-        .replace('\n', "\r\n");
+    let encr = pgp::symm_encrypt(passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -286,7 +284,7 @@ pub async fn continue_key_transfer(
         let file = open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
         let armored_key = decrypt_setup_file(&sc, file).await?;
-        set_self_key(context, &armored_key, true).await?;
+        set_self_key(context, &armored_key, true, true).await?;
         maybe_add_bcc_self_device_msg(context).await?;
 
         Ok(())
@@ -295,32 +293,35 @@ pub async fn continue_key_transfer(
     }
 }
 
-async fn set_self_key(context: &Context, armored: &str, set_default: bool) -> Result<()> {
+async fn set_self_key(
+    context: &Context,
+    armored: &str,
+    set_default: bool,
+    prefer_encrypt_required: bool,
+) -> Result<()> {
     // try hard to only modify key-state
     let (private_key, header) = SignedSecretKey::from_asc(armored)?;
     let public_key = private_key.split_public_key()?;
-    if let Some(preferencrypt) = header.get("Autocrypt-Prefer-Encrypt") {
-        let e2ee_enabled = match preferencrypt.as_str() {
-            "nopreference" => 0,
-            "mutual" => 1,
-            _ => {
-                bail!("invalid Autocrypt-Prefer-Encrypt header: {:?}", header);
+    let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
+    match preferencrypt.map(|s| s.as_str()) {
+        Some(headerval) => {
+            let e2ee_enabled = match headerval {
+                "nopreference" => 0,
+                "mutual" => 1,
+                _ => {
+                    bail!("invalid Autocrypt-Prefer-Encrypt header: {:?}", header);
+                }
+            };
+            context
+                .sql
+                .set_raw_config_int("e2ee_enabled", e2ee_enabled)
+                .await?;
+        }
+        None => {
+            if prefer_encrypt_required {
+                bail!("missing Autocrypt-Prefer-Encrypt header");
             }
-        };
-        context
-            .sql
-            .set_raw_config_int("e2ee_enabled", e2ee_enabled)
-            .await?;
-    } else {
-        // `Autocrypt-Prefer-Encrypt` is not included
-        // in keys exported to file.
-        //
-        // `Autocrypt-Prefer-Encrypt` also SHOULD be sent
-        // in Autocrypt Setup Message according to Autocrypt specification,
-        // but K-9 6.802 does not include this header.
-        //
-        // We keep current setting in this case.
-        info!(context, "No Autocrypt-Prefer-Encrypt header.");
+        }
     };
 
     let self_addr = context.get_primary_self_addr().await?;
@@ -603,7 +604,7 @@ async fn export_backup_inner(
 async fn import_secret_key(context: &Context, path: &Path, set_default: bool) -> Result<()> {
     let buf = read_file(context, &path).await?;
     let armored = std::string::String::from_utf8_lossy(&buf);
-    set_self_key(context, &armored, set_default).await?;
+    set_self_key(context, &armored, set_default, false).await?;
     Ok(())
 }
 
@@ -824,7 +825,6 @@ mod tests {
 
     use super::*;
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
-    use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
     use crate::test_utils::{alice_keypair, TestContext, TestContextManager};
 
@@ -834,17 +834,15 @@ mod tests {
         let msg = render_setup_file(&t, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
+        // In particular note the mixing of `\r\n` and `\n` depending
+        // on who generated the strings.
         assert!(msg.contains("<title>Autocrypt Setup Message</title"));
         assert!(msg.contains("<h1>Autocrypt Setup Message</h1>"));
         assert!(msg.contains("<p>This is the Autocrypt Setup Message used to"));
         assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
         assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
-        assert!(msg.contains("Passphrase-Begin: he\r\n"));
-        assert!(msg.contains("-----END PGP MESSAGE-----\r\n"));
-
-        for line in msg.rsplit_terminator('\n') {
-            assert!(line.ends_with('\r'));
-        }
+        assert!(msg.contains("Passphrase-Begin: he\n"));
+        assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1195,22 +1193,4 @@ mod tests {
 
         Ok(())
     }
-
-    /// Tests reception of Autocrypt Setup Message from K-9 6.802.
-    ///
-    /// Unlike Autocrypt Setup Message sent by Delta Chat,
-    /// this message does not contain `Autocrypt-Prefer-Encrypt` header.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_key_transfer_k_9() -> Result<()> {
-        let t = &TestContext::new().await;
-        t.configure_addr("autocrypt@nine.testrun.org").await;
-
-        let raw = include_bytes!("../test-data/message/k-9-autocrypt-setup-message.eml");
-        let received = receive_imf(t, raw, false).await?.unwrap();
-
-        let setup_code = "0655-9868-8252-5455-4232-5158-1237-5333-2638";
-        continue_key_transfer(t, *received.msg_ids.last().unwrap(), setup_code).await?;
-
-        Ok(())
-    }
 }

src/imex/transfer.rs

@@ -656,12 +656,6 @@ mod tests {
         let text = fs::read_to_string(&path).await.unwrap();
         assert_eq!(text, "i am attachment");
 
-        let path = path.with_file_name("saved.txt");
-        msg.save_file(&ctx1, &path).await.unwrap();
-        let text = fs::read_to_string(&path).await.unwrap();
-        assert_eq!(text, "i am attachment");
-        assert!(msg.save_file(&ctx1, &path).await.is_err());
-
         // Check that both received the ImexProgress events.
         ctx0.evtracker
             .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000)))

src/location.rs

@@ -1,14 +1,4 @@
 //! Location handling.
-//!
-//! Delta Chat handles two kind of locations.
-//!
-//! There are two kinds of locations:
-//! - Independent locations, also known as Points of Interest (POI).
-//! - Path locations.
-//!
-//! Locations are sent as KML attachments.
-//! Independent locations are sent in `message.kml` attachments
-//! and path locations are sent in `location.kml` attachments.
 
 use std::time::Duration;
 
@@ -18,7 +8,6 @@ use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 use tokio::time::timeout;
 
 use crate::chat::{self, ChatId};
-use crate::constants::DC_CHAT_ID_TRASH;
 use crate::contact::ContactId;
 use crate::context::Context;
 use crate::events::EventType;
@@ -361,7 +350,6 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
         )
         .await?;
 
-    let mut stored_location = false;
     for chat_id in chats {
         context.sql.execute(
                 "INSERT INTO locations  \
@@ -374,7 +362,6 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
                     chat_id,
                     ContactId::SELF,
                 )).await.context("Failed to store location")?;
-        stored_location = true;
 
         info!(context, "Stored location for chat {chat_id}.");
         continue_streaming = true;
@@ -382,10 +369,6 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
     if continue_streaming {
         context.emit_location_changed(Some(ContactId::SELF)).await?;
     };
-    if stored_location {
-        // Interrupt location loop so it may send a location-only message.
-        context.scheduler.interrupt_location().await;
-    }
 
     Ok(continue_streaming)
 }
@@ -478,58 +461,6 @@ pub async fn delete_all(context: &Context) -> Result<()> {
     Ok(())
 }
 
-/// Deletes expired locations.
-///
-/// Only path locations are deleted.
-/// POIs should be deleted when corresponding message is deleted.
-pub(crate) async fn delete_expired(context: &Context, now: i64) -> Result<()> {
-    let Some(delete_device_after) = context.get_config_delete_device_after().await? else {
-        return Ok(());
-    };
-
-    let threshold_timestamp = now.saturating_sub(delete_device_after);
-    let deleted = context
-        .sql
-        .execute(
-            "DELETE FROM locations WHERE independent=0 AND timestamp < ?",
-            (threshold_timestamp,),
-        )
-        .await?
-        > 0;
-    if deleted {
-        info!(context, "Deleted {deleted} expired locations.");
-        context.emit_location_changed(None).await?;
-    }
-    Ok(())
-}
-
-/// Deletes location if it is an independent location.
-///
-/// This function is used when a message is deleted
-/// that has a corresponding `location_id`.
-pub(crate) async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
-    context
-        .sql
-        .execute(
-            "DELETE FROM locations WHERE independent = 1 AND id=?",
-            (location_id as i32,),
-        )
-        .await?;
-    Ok(())
-}
-
-/// Deletes POI locations that don't have corresponding message anymore.
-pub(crate) async fn delete_orphaned_poi_locations(context: &Context) -> Result<()> {
-    context.sql.execute("
-    DELETE FROM locations
-    WHERE independent=1 AND id NOT IN
-    (SELECT location_id from MSGS LEFT JOIN locations
-     ON locations.id=location_id
-     WHERE location_id>0 -- This check makes the query faster by not looking for locations with ID 0 that don't exist.
-     AND msgs.chat_id != ?)", (DC_CHAT_ID_TRASH,)).await?;
-    Ok(())
-}
-
 /// Returns `location.kml` contents.
 pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<Option<(String, u32)>> {
     let mut last_added_location_id = 0;
@@ -884,10 +815,8 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
-    use crate::config::Config;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::{TestContext, TestContextManager};
-    use crate::tools::SystemTime;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_kml_parse() {
@@ -1037,8 +966,6 @@ Content-Disposition: attachment; filename="location.kml"
         let mut msg = Message::new(Viewtype::Image);
         msg.set_file(file.to_str().unwrap(), None);
         let sent = alice.send_msg(alice_chat.id, &mut msg).await;
-        let alice_msg = Message::load_from_db(&alice, sent.sender_msg_id).await?;
-        assert_eq!(alice_msg.has_location(), false);
 
         let msg = bob.recv_msg_opt(&sent).await.unwrap();
         assert!(msg.chat_id == bob_chat_id);
@@ -1047,60 +974,6 @@ Content-Disposition: attachment; filename="location.kml"
         let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.first().unwrap()).await?;
         assert_eq!(bob_msg.chat_id, bob_chat_id);
         assert_eq!(bob_msg.viewtype, Viewtype::Image);
-        assert_eq!(bob_msg.has_location(), false);
-
-        let bob_locations = get_range(&bob, None, None, 0, 0).await?;
-        assert_eq!(bob_locations.len(), 1);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_delete_expired_locations() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-
-        // Alice enables deletion of messages from device after 1 week.
-        alice
-            .set_config(Config::DeleteDeviceAfter, Some("604800"))
-            .await?;
-        // Bob enables deletion of messages from device after 1 day.
-        bob.set_config(Config::DeleteDeviceAfter, Some("86400"))
-            .await?;
-
-        let alice_chat = alice.create_chat(bob).await;
-
-        // Alice enables location streaming.
-        // Bob receives a message saying that Alice enabled location streaming.
-        send_locations_to_chat(alice, alice_chat.id, 60).await?;
-        bob.recv_msg(&alice.pop_sent_msg().await).await;
-
-        // Alice gets new location from GPS.
-        assert_eq!(set(alice, 10.0, 20.0, 1.0).await?, true);
-        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
-
-        // 10 seconds later location sending stream manages to send location.
-        SystemTime::shift(Duration::from_secs(10));
-        delete_expired(alice, time()).await?;
-        maybe_send_locations(alice).await?;
-        bob.recv_msg_opt(&alice.pop_sent_msg().await).await;
-        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
-        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 1);
-
-        // Day later Bob removes location.
-        SystemTime::shift(Duration::from_secs(86400));
-        delete_expired(alice, time()).await?;
-        delete_expired(bob, time()).await?;
-        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 1);
-        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 0);
-
-        // Week late Alice removes location.
-        SystemTime::shift(Duration::from_secs(604800));
-        delete_expired(alice, time()).await?;
-        delete_expired(bob, time()).await?;
-        assert_eq!(get_range(alice, None, None, 0, 0).await?.len(), 0);
-        assert_eq!(get_range(bob, None, None, 0, 0).await?.len(), 0);
 
         Ok(())
     }

src/message.rs

@@ -6,10 +6,9 @@ use std::path::{Path, PathBuf};
 use anyhow::{ensure, format_err, Context as _, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
-use tokio::{fs, io};
 
 use crate::blob::BlobObject;
-use crate::chat::{Chat, ChatId, ChatIdBlocked};
+use crate::chat::{Chat, ChatId};
 use crate::chatlist_events;
 use crate::config::Config;
 use crate::constants::{
@@ -22,7 +21,6 @@ use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::imap::markseen_on_imap_table;
-use crate::location::delete_poi_location;
 use crate::mimeparser::{parse_message_id, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
@@ -607,19 +605,6 @@ impl Message {
         self.param.get_path(Param::File, context).unwrap_or(None)
     }
 
-    /// Save file copy at the user-provided path.
-    pub async fn save_file(&self, context: &Context, path: &Path) -> Result<()> {
-        let path_src = self.get_file(context).context("No file")?;
-        let mut src = fs::OpenOptions::new().read(true).open(path_src).await?;
-        let mut dst = fs::OpenOptions::new()
-            .write(true)
-            .create_new(true)
-            .open(path)
-            .await?;
-        io::copy(&mut src, &mut dst).await?;
-        Ok(())
-    }
-
     /// If message is an image or gif, set Param::Width and Param::Height
     pub(crate) async fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<()> {
         if self.viewtype.has_file() {
@@ -655,11 +640,13 @@ impl Message {
         Ok(())
     }
 
-    /// Check if a message has a POI location bound to it.
-    /// These locations are also returned by [`location::get_range()`].
-    /// The UI may decide to display a special icon beside such messages.
+    /// Check if a message has a location bound to it.
+    /// These messages are also returned by get_locations()
+    /// and the UI may decide to display a special icon beside such messages,
     ///
-    /// [`location::get_range()`]: crate::location::get_range
+    /// @memberof Message
+    /// @param msg The message object.
+    /// @return 1=Message has location bound to it, 0=No location bound to message.
     pub fn has_location(&self) -> bool {
         self.location_id != 0
     }
@@ -669,17 +656,13 @@ impl Message {
     /// at a position different from the self-location.
     /// You should not call this function
     /// if you want to bind the current self-location to a message;
-    /// this is done by [`location::set()`] and [`send_locations_to_chat()`].
+    /// this is done by set_location() and send_locations_to_chat().
     ///
-    /// Typically results in the event [`LocationChanged`] with
-    /// `contact_id` set to [`ContactId::SELF`].
+    /// Typically results in the event #DC_EVENT_LOCATION_CHANGED with
+    /// contact_id set to ContactId::SELF.
     ///
-    /// `latitude` is the North-south position of the location.
-    /// `longitutde` is the East-west position of the location.
-    ///
-    /// [`location::set()`]: crate::location::set
-    /// [`send_locations_to_chat()`]: crate::location::send_locations_to_chat
-    /// [`LocationChanged`]: crate::events::EventType::LocationChanged
+    /// @param latitude North-south position of the location.
+    /// @param longitude East-west position of the location.
     pub fn set_location(&mut self, latitude: f64, longitude: f64) {
         if latitude == 0.0 && longitude == 0.0 {
             return;
@@ -1060,7 +1043,6 @@ impl Message {
         filemime: Option<&str>,
     ) -> Result<()> {
         let blob = BlobObject::create(context, suggested_name, data).await?;
-        self.param.set(Param::Filename, suggested_name);
         self.param.set(Param::File, blob.as_name());
         self.param.set_optional(Param::MimeType, filemime);
         Ok(())
@@ -1130,7 +1112,7 @@ impl Message {
                 .get_bool(Param::GuaranteeE2ee)
                 .unwrap_or_default()
             {
-                self.param.set(Param::ProtectQuote, "1");
+                self.param.set(Param::GuaranteeE2ee, "1");
             }
 
             let text = quote.get_text();
@@ -1574,6 +1556,17 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
     Ok(())
 }
 
+async fn delete_poi_location(context: &Context, location_id: u32) -> Result<()> {
+    context
+        .sql
+        .execute(
+            "DELETE FROM locations WHERE independent = 1 AND id=?;",
+            (location_id as i32,),
+        )
+        .await?;
+    Ok(())
+}
+
 /// Marks requested messages as seen.
 pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()> {
     if msg_ids.is_empty() {
@@ -1813,9 +1806,8 @@ pub async fn estimate_deletion_cnt(
     from_server: bool,
     seconds: i64,
 ) -> Result<usize> {
-    let self_chat_id = ChatIdBlocked::lookup_by_contact(context, ContactId::SELF)
+    let self_chat_id = ChatId::lookup_by_contact(context, ContactId::SELF)
         .await?
-        .map(|c| c.id)
         .unwrap_or_default();
     let threshold_timestamp = time() - seconds;
 
@@ -2008,11 +2000,8 @@ mod tests {
     use num_traits::FromPrimitive;
 
     use super::*;
-    use crate::chat::{
-        self, add_contact_to_chat, marknoticed_chat, send_text_msg, ChatItem, ProtectionStatus,
-    };
+    use crate::chat::{self, marknoticed_chat, send_text_msg, ChatItem};
     use crate::chatlist::Chatlist;
-    use crate::config::Config;
     use crate::reaction::send_reaction;
     use crate::receive_imf::receive_imf;
     use crate::test_utils as test;
@@ -2036,6 +2025,8 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_prepare_message_and_send() {
+        use crate::config::Config;
+
         let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
@@ -2174,6 +2165,8 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_quote() {
+        use crate::config::Config;
+
         let d = test::TestContext::new().await;
         let ctx = &d.ctx;
 
@@ -2206,42 +2199,6 @@ mod tests {
         assert_eq!(quoted_msg.get_text(), msg2.quoted_text().unwrap());
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_unencrypted_quote_encrypted_message() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-
-        let alice_group = alice
-            .create_group_with_members(ProtectionStatus::Unprotected, "Group chat", &[bob])
-            .await;
-        let sent = alice.send_text(alice_group, "Hi! I created a group").await;
-        let bob_received_message = bob.recv_msg(&sent).await;
-
-        let bob_group = bob_received_message.chat_id;
-        bob_group.accept(bob).await?;
-        let sent = bob.send_text(bob_group, "Encrypted message").await;
-        let alice_received_message = alice.recv_msg(&sent).await;
-        assert!(alice_received_message.get_showpadlock());
-
-        // Alice adds contact without key so chat becomes unencrypted.
-        let alice_flubby_contact_id =
-            Contact::create(alice, "Flubby", "flubby@example.org").await?;
-        add_contact_to_chat(alice, alice_group, alice_flubby_contact_id).await?;
-
-        // Alice quotes encrypted message in unencrypted chat.
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_quote(alice, Some(&alice_received_message)).await?;
-        chat::send_msg(alice, alice_group, &mut msg).await?;
-
-        let bob_received_message = bob.recv_msg(&alice.pop_sent_msg().await).await;
-        assert_eq!(bob_received_message.quoted_text().unwrap(), "...");
-        assert_eq!(bob_received_message.get_showpadlock(), false);
-
-        Ok(())
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_get_chat_id() {
         // Alice receives a message that pops up as a contact request

src/mimefactory.rs

@@ -669,19 +669,19 @@ impl<'a> MimeFactory<'a> {
 
         let mut is_gossiped = false;
 
-        let peerstates = self.peerstates_for_recipients(context).await?;
-        let should_encrypt =
-            encrypt_helper.should_encrypt(context, e2ee_guaranteed, &peerstates)?;
-        let is_encrypted = should_encrypt && !force_plaintext;
-
         let (main_part, parts) = match self.loaded {
             Loaded::Message { .. } => {
-                self.render_message(context, &mut headers, &grpimage, is_encrypted)
+                self.render_message(context, &mut headers, &grpimage)
                     .await?
             }
             Loaded::Mdn { .. } => (self.render_mdn(context).await?, Vec::new()),
         };
 
+        let peerstates = self.peerstates_for_recipients(context).await?;
+        let should_encrypt =
+            encrypt_helper.should_encrypt(context, e2ee_guaranteed, &peerstates)?;
+        let is_encrypted = should_encrypt && !force_plaintext;
+
         let message = if parts.is_empty() {
             // Single part, render as regular message.
             main_part
@@ -960,7 +960,6 @@ impl<'a> MimeFactory<'a> {
         context: &Context,
         headers: &mut MessageHeaders,
         grpimage: &Option<String>,
-        is_encrypted: bool,
     ) -> Result<(PartBuilder, Vec<PartBuilder>)> {
         let chat = match &self.loaded {
             Loaded::Message { chat } => chat,
@@ -1222,16 +1221,6 @@ impl<'a> MimeFactory<'a> {
             .msg
             .quoted_text()
             .map(|quote| format_flowed_quote(&quote) + "\r\n\r\n");
-        if !is_encrypted
-            && self
-                .msg
-                .param
-                .get_bool(Param::ProtectQuote)
-                .unwrap_or_default()
-        {
-            // Message is not encrypted but quotes encrypted message.
-            quoted_text = Some("> ...\r\n\r\n".to_string());
-        }
         if quoted_text.is_none() && final_text.starts_with('>') {
             // Insert empty line to avoid receiver treating user-sent quote as topquote inserted by
             // Delta Chat.

src/mimeparser.rs

@@ -415,6 +415,8 @@ impl MimeMessage {
             if let (Some(peerstate), Ok(mail)) = (&mut decryption_info.peerstate, mail) {
                 if timestamp_sent > peerstate.last_seen_autocrypt
                     && mail.ctype.mimetype != "multipart/report"
+                // Disallowing keychanges is disabled for now:
+                // && decryption_info.dkim_results.allow_keychange
                 {
                     peerstate.degrade_encryption(timestamp_sent);
                 }
@@ -504,6 +506,13 @@ impl MimeMessage {
         parser.heuristically_parse_ndn(context).await;
         parser.parse_headers(context).await?;
 
+        // Disallowing keychanges is disabled for now
+        // if !decryption_info.dkim_results.allow_keychange {
+        //     for part in parser.parts.iter_mut() {
+        //         part.error = Some("Seems like DKIM failed, this either is an attack or (more likely) a bug in Authentication-Results checking. Please tell us about this at https://support.delta.chat.".to_string());
+        //     }
+        // }
+
         if parser.is_mime_modified {
             parser.decoded_data = mail_raw;
         }
@@ -653,34 +662,32 @@ impl MimeMessage {
             self.squash_attachment_parts();
         }
 
-        if !context.get_config_bool(Config::Bot).await? {
-            if let Some(ref subject) = self.get_subject() {
-                let mut prepend_subject = true;
-                if !self.decrypting_failed {
-                    let colon = subject.find(':');
-                    if colon == Some(2)
-                        || colon == Some(3)
-                        || self.has_chat_version()
-                        || subject.contains("Chat:")
-                    {
-                        prepend_subject = false
-                    }
+        if let Some(ref subject) = self.get_subject() {
+            let mut prepend_subject = true;
+            if !self.decrypting_failed {
+                let colon = subject.find(':');
+                if colon == Some(2)
+                    || colon == Some(3)
+                    || self.has_chat_version()
+                    || subject.contains("Chat:")
+                {
+                    prepend_subject = false
                 }
+            }
 
-                // For mailing lists, always add the subject because sometimes there are different topics
-                // and otherwise it might be hard to keep track:
-                if self.is_mailinglist_message() && !self.has_chat_version() {
-                    prepend_subject = true;
-                }
+            // For mailing lists, always add the subject because sometimes there are different topics
+            // and otherwise it might be hard to keep track:
+            if self.is_mailinglist_message() && !self.has_chat_version() {
+                prepend_subject = true;
+            }
 
-                if prepend_subject && !subject.is_empty() {
-                    let part_with_text = self
-                        .parts
-                        .iter_mut()
-                        .find(|part| !part.msg.is_empty() && !part.is_reaction);
-                    if let Some(part) = part_with_text {
-                        part.msg = format!("{} – {}", subject, part.msg);
-                    }
+            if prepend_subject && !subject.is_empty() {
+                let part_with_text = self
+                    .parts
+                    .iter_mut()
+                    .find(|part| !part.msg.is_empty() && !part.is_reaction);
+                if let Some(part) = part_with_text {
+                    part.msg = format!("{} – {}", subject, part.msg);
                 }
             }
         }
@@ -3926,31 +3933,4 @@ Content-Disposition: reaction\n\
 
         Ok(())
     }
-
-    /// Tests that subject is not prepended to the message
-    /// when bot receives it.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_bot_no_subject() {
-        let context = TestContext::new().await;
-        context.set_config(Config::Bot, Some("1")).await.unwrap();
-        let raw = br#"Message-ID: <foobar@example.org>
-From: foo <foo@example.org>
-Subject: Some subject
-To: bar@example.org
-MIME-Version: 1.0
-Content-Type: text/plain; charset=utf-8
-
-/help
-"#;
-
-        let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
-            .await
-            .unwrap();
-        assert_eq!(message.get_subject(), Some("Some subject".to_string()));
-
-        assert_eq!(message.parts.len(), 1);
-        assert_eq!(message.parts[0].typ, Viewtype::Text);
-        // Not "Some subject – /help"
-        assert_eq!(message.parts[0].msg, "/help");
-    }
 }

src/param.rs

@@ -48,11 +48,6 @@ pub enum Param {
     /// For Messages: message is encrypted, outgoing: guarantee E2EE or the message is not send
     GuaranteeE2ee = b'c',
 
-    /// For Messages: quoted message is encrypted.
-    ///
-    /// If this message is sent unencrypted, quote text should be replaced.
-    ProtectQuote = b'0',
-
     /// For Messages: decrypted with validation errors or without mutual set, if neither
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',

src/receive_imf.rs

@@ -1669,10 +1669,11 @@ async fn save_locations(
         if let Some(addr) = &location_kml.addr {
             let contact = Contact::get_by_id(context, from_id).await?;
             if contact.get_addr().to_lowercase() == addr.to_lowercase() {
-                if location::save(context, chat_id, from_id, &location_kml.locations, false)
-                    .await?
-                    .is_some()
+                if let Some(newest_location_id) =
+                    location::save(context, chat_id, from_id, &location_kml.locations, false)
+                        .await?
                 {
+                    location::set_msg_location_id(context, msg_id, newest_location_id).await?;
                     send_event = true;
                 }
             } else {

src/receive_imf/tests.rs

@@ -2994,15 +2994,11 @@ async fn test_long_and_duplicated_filenames() -> Result<()> {
             let resulting_filename = msg.get_filename().unwrap();
             assert_eq!(resulting_filename, filename);
             let path = msg.get_file(t).unwrap();
-            let path2 = path.with_file_name("saved.txt");
-            msg.save_file(t, &path2).await.unwrap();
             assert!(
                 path.to_str().unwrap().ends_with(".tar.gz"),
                 "path {path:?} doesn't end with .tar.gz"
             );
-            assert_eq!(fs::read_to_string(&path).await.unwrap(), content);
-            assert_eq!(fs::read_to_string(&path2).await.unwrap(), content);
-            fs::remove_file(path2).await.unwrap();
+            assert_eq!(fs::read_to_string(path).await.unwrap(), content);
         }
         check_message(&msg_alice, &alice, filename_sent, &content).await;
         check_message(&msg_bob, &bob, filename_sent, &content).await;

src/sql.rs

@@ -15,7 +15,6 @@ use crate::context::Context;
 use crate::debug_logging::set_debug_logging_xdc;
 use crate::ephemeral::start_ephemeral_timers;
 use crate::imex::BLOBS_BACKUP_NAME;
-use crate::location::delete_orphaned_poi_locations;
 use crate::log::LogExt;
 use crate::message::{Message, MsgId, Viewtype};
 use crate::param::{Param, Params};
@@ -780,14 +779,6 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
-    // Delete POI locations
-    // which don't have corresponding message.
-    delete_orphaned_poi_locations(context)
-        .await
-        .context("Failed to delete orphaned POI locations")
-        .log_err(context)
-        .ok();
-
     info!(context, "Housekeeping done.");
     Ok(())
 }

src/sql/migrations.rs

@@ -613,7 +613,6 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid);
         ).await?;
     }
     if dbversion < 93 {
-        // `sending_domains` is now unused, but was not removed for backwards compatibility.
         sql.execute_migration(
             "CREATE TABLE sending_domains(domain TEXT PRIMARY KEY, dkim_works INTEGER DEFAULT 0);",
             93,

src/tools.rs

@@ -683,12 +683,7 @@ pub(crate) fn buf_decompress(buf: &[u8]) -> Result<Vec<u8>> {
 mod tests {
     #![allow(clippy::indexing_slicing)]
 
-    use chrono::NaiveDate;
-    use proptest::prelude::*;
-
     use super::*;
-    use crate::chatlist::Chatlist;
-    use crate::{chat, test_utils};
     use crate::{receive_imf::receive_imf, test_utils::TestContext};
 
     #[test]
@@ -735,7 +730,7 @@ Message-ID: 2dfdbde7@example.org
 Hop: From: localhost; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:22 +0000
 Hop: From: hq5.merlinux.eu; By: hq5.merlinux.eu; Date: Sat, 14 Sep 2019 17:00:25 +0000
 
-DKIM Results: Passed=true";
+DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
         check_parse_receive_headers_integration(raw, expected).await;
 
         let raw = include_bytes!("../test-data/message/encrypted_with_received_headers.eml");
@@ -754,7 +749,7 @@ Hop: From: [127.0.0.1]; By: mail.example.org; Date: Mon, 27 Dec 2021 11:21:21 +0
 Hop: From: mout.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
 Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 +0000
 
-DKIM Results: Passed=true";
+DKIM Results: Passed=true, Works=true, Allow_Keychange=true";
         check_parse_receive_headers_integration(raw, expected).await;
     }
 
@@ -966,6 +961,12 @@ DKIM Results: Passed=true";
         assert!(extract_grpid_from_rfc724_mid(mid.as_str()).is_none());
     }
 
+    use chrono::NaiveDate;
+    use proptest::prelude::*;
+
+    use crate::chatlist::Chatlist;
+    use crate::{chat, test_utils};
+
     proptest! {
         #[test]
         fn test_truncate(

src/webxdc.rs

@@ -862,7 +862,6 @@ impl Message {
 mod tests {
     use std::time::Duration;
 
-    use regex::Regex;
     use serde_json::json;
 
     use super::*;
@@ -1712,6 +1711,8 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_render_webxdc_status_update_object_range() -> Result<()> {
+        use regex::Regex;
+
         let t = TestContext::new_alice().await;
         let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat").await?;
         let instance = send_webxdc_instance(&t, chat_id).await?;

test-data/message/k-9-autocrypt-setup-message.eml

@@ -1,65 +0,0 @@
-Return-Path: <autocrypt@nine.testrun.org>
-Delivered-To: autocrypt@nine.testrun.org
-Received: from nine.testrun.org
-	by nine with LMTP
-	id wNinAKX2J2YWDwEAPdT8mA
-	(envelope-from <autocrypt@nine.testrun.org>)
-	for <autocrypt@nine.testrun.org>; Tue, 23 Apr 2024 19:57:57 +0200
-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=nine.testrun.org;
-	s=opendkim; t=1713895076;
-	bh=yuHuHSbYX5hE/xr8aU2fy/SlqfTL7XjfV2m1eEePTz4=;
-	h=Subject:Date:From:To:From;
-	b=ZbVNpJ8zjHmgrCqiRnqzENcR/PwR/G182hL18U5bp5CZmkyWcuhQU0EkhkJpCCv1n
-	 8bZ9WlOT0cmzBHpWU43t7HufuUM56NwwuVqEuz2agpVzQV8zKIPhthrBzbYIeR4Prg
-	 1DgwWr8EhotoV6yPgzxi9sMyO3l4spJeaREisB5MPOIdKeIxtRPLR+Woo5hQWNTFoh
-	 ZQtCcY7w5vxXGhBMVPXOjbrrzOCsE5gGB5QYSAR8Bv3ZdJn/mHvIRCEJG5hJGSxXjQ
-	 fD0UGJ5m5RVrF0tWnZ7U5tpoRD/UVV1+Us9Woq733R97ZchpoE4hNpMG9zYW90z4QU
-	 kBajbsH81Nm0A==
-MIME-Version: 1.0
-Content-Type: multipart/mixed;
- boundary=----X2OJUZLGILKJEHMTO29ZMST9701ZDH
-Content-Transfer-Encoding: 7bit
-Subject: Autocrypt Setup Message
-Autocrypt-Setup-Message: v1
-Date: Tue, 23 Apr 2024 19:57:57 +0200
-From: autocrypt@nine.testrun.org
-To: autocrypt@nine.testrun.org
-Message-Id: <20240423175756.F19EB17C214A@nine.testrun.org>
-
-------X2OJUZLGILKJEHMTO29ZMST9701ZDH
-Content-Type: text/plain;
- charset=utf-8
-Content-Transfer-Encoding: quoted-printable
-
-This message contains all information to transfer your Autocrypt settings a=
-long with your secret key securely from your original device=2E To set up y=
-our new device for Autocrypt, please follow the instructions that should be=
- presented by your new device=2E You can keep this message and use it as a =
-backup for your secret key=2E If you want to do this, you should write down=
- the password and store it securely=2E
-------X2OJUZLGILKJEHMTO29ZMST9701ZDH
-Content-Transfer-Encoding: 7bit
-Content-Type: application/autocrypt-setup
-Content-Disposition: attachment; filename="autocrypt-setup-message"
-
------BEGIN PGP MESSAGE-----
-Passphrase-Format: numeric9x4
-Passphrase-Begin: 06
-
-ww0ECQMCAhlJ+TRwb2Fg0sGXAUc+92rmg4k57Sd4D3O/SPQNzShbVdlKsoFzyH+B
-YhimOr/8C5ZHyg/WjRGlk4pD+t57WfVdE7LYnv8qsK86h2kffZAGlj+B9Lh9+qbV
-KgJLpHUKg7ZGa/9aMq7KuFoNSNTbcHtzJ/Ml9GVe+opimER87mpFCjmaEHCcCp0a
-ZeS5VU8gTV7AKuPW40BBipyEmKpUvE/ZWfz3KSI4RZyIwM8v8kXBMojT4WLqWm93
-JoEKUyeh+3JKMvsfyRbmHXrHprG9f2e8PLvNkAiie68YJniFnwA8nmNSnPv9S9rf
-7oUHtnTDKJ4FIpmfPgj1v/KIWWW9KaZWHi7K5mFUCTb4pBoCRIGaFh+JzbSlNL9i
-fz7HIiN95bFJ4xXXL4gcU9wO5//npkVDUncaeHhUy1VBLu0NFYvze+s+eAIesqec
-X3x++U9d+Slbpa1G2Z5Knj50mBY+k9aNwVMZGu50hzhPvdwesqmbr+GTSh0O1bxI
-gw/cDq5s58Ewze3WvYaLxJz/RcwOCGSV8k21FM4WTnEahs4yfLbzNuusYvvciU6l
-w0eZC+vEmh+bINSSRX/mcvkQcIkkCsqvfWyxdSNIBCwmR86oalWnxZniBLbbbZHD
-0KAsv0w7t00Y715gyyFWyiEiT5Lyl4TA+cUIHKmmpKOaVubz50UD1z5rqT7joJ7G
-KRmWtQW8MScgcmK7+tyavLQOxwe8i8i9JkUy+d9jhj17XZil/If26Q3V3epqCXq3
-FdvEvvNGJF0DyJ4YAe9QMBumf22sMmX/XVock9/k0pB46mciMhPL3VA=
-=LYx9
------END PGP MESSAGE-----
-
-------X2OJUZLGILKJEHMTO29ZMST9701ZDH--