chore(release): prepare for 1.155.4

this converts old QR codes to the new format, in an hacky, but simple
way, see #6518 for more details and for code snippet


then QR code change is esp. bad as ppl will have different versions for
some days at least, weakening overall UX, esp. of first-time-users that
may come to delta because of praised, seamless multidevice ... :)


i tested in https://github.com/deltachat/deltachat-ios/pull/2595 that
this actually fixes the problem, and there is no deeper issue with
changed chashes or so - seemed not to be the case, at least, with this
hack, core accepts QR codes from the released 1.52-and-before series

this hack gives user time to update, it can be removed after some months
(we can also remove the old BACKUP qr code alltogether then)

we should still not wait too long with the PR as there are already
versions out with the "new/bad" QR code (and growing, as new iOS
installations will get the new format, one cannot revert a version, only
pause rollout)

---------

Co-authored-by: link2xt <link2xt@testrun.org>

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.82.0
+      RUSTUP_TOOLCHAIN: 1.84.1
     steps:
       - uses: actions/checkout@v4
         with:
@@ -37,8 +37,10 @@ jobs:
         run: cargo fmt --all -- --check
       - name: Run clippy
         run: scripts/clippy.sh
-      - name: Check
+      - name: Check with all features
         run: cargo check --workspace --all-targets --all-features
+      - name: Check with only default features
+        run: cargo check --all-targets
 
   npm_constants:
     name: Check if node constants are up to date
@@ -95,15 +97,15 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.82.0
+            rust: 1.84.1
           - os: windows-latest
-            rust: 1.82.0
+            rust: 1.84.1
           - os: macos-latest
-            rust: 1.82.0
+            rust: 1.84.1
 
-          # Minimum Supported Rust Version = 1.77.0
+          # Minimum Supported Rust Version = 1.81.0
           - os: ubuntu-latest
-            rust: 1.77.0
+            rust: 1.81.0
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -150,7 +152,7 @@ jobs:
         uses: swatinem/rust-cache@v2
 
       - name: Build C library
-        run: cargo build -p deltachat_ffi --features jsonrpc
+        run: cargo build -p deltachat_ffi
 
       - name: Upload C library
         uses: actions/upload-artifact@v4
@@ -221,11 +223,11 @@ jobs:
           - os: macos-latest
             python: pypy3.10
 
-          # Minimum Supported Python Version = 3.7
+          # Minimum Supported Python Version = 3.8
           # This is the minimum version for which manylinux Python wheels are
           # built. Test it with minimum supported Rust version.
           - os: ubuntu-latest
-            python: 3.7
+            python: 3.8
 
     runs-on: ${{ matrix.os }}
     steps:
@@ -275,9 +277,9 @@ jobs:
           - os: macos-latest
             python: pypy3.10
 
-          # Minimum Supported Python Version = 3.7
+          # Minimum Supported Python Version = 3.8
           - os: ubuntu-latest
-            python: 3.7
+            python: 3.8
 
     runs-on: ${{ matrix.os }}
     steps:

.github/workflows/dependabot.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v2.2.0
+        uses: dependabot/fetch-metadata@v2.3.0
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
       - name: Approve a PR

.github/workflows/nix.yml

@@ -0,0 +1,104 @@
+name: Test Nix flake
+
+on:
+  pull_request:
+    paths:
+      - flake.nix
+      - flake.lock
+  push:
+    paths:
+      - flake.nix
+      - flake.lock
+    branches:
+      - main
+
+jobs:
+  format:
+    name: check flake formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix fmt
+
+      # Check that formatting does not change anything.
+      - run: git diff --exit-code
+
+  build:
+    name: nix build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        installable:
+          # Ensure `nix develop` will work.
+          - devShells.x86_64-linux.default
+
+          - deltachat-python
+          - deltachat-repl
+          - deltachat-repl-aarch64-linux
+          - deltachat-repl-arm64-v8a-android
+          - deltachat-repl-armeabi-v7a-android
+          - deltachat-repl-armv6l-linux
+          - deltachat-repl-armv7l-linux
+          - deltachat-repl-i686-linux
+          - deltachat-repl-win32
+          - deltachat-repl-win64
+          - deltachat-repl-x86_64-linux
+          - deltachat-rpc-client
+          - deltachat-rpc-server
+          - deltachat-rpc-server-aarch64-linux
+          - deltachat-rpc-server-aarch64-linux-wheel
+          - deltachat-rpc-server-arm64-v8a-android
+          - deltachat-rpc-server-armeabi-v7a-android
+          - deltachat-rpc-server-armv6l-linux
+          - deltachat-rpc-server-armv6l-linux-wheel
+          - deltachat-rpc-server-armv7l-linux
+          - deltachat-rpc-server-armv7l-linux-wheel
+          - deltachat-rpc-server-i686-linux
+          - deltachat-rpc-server-i686-linux-wheel
+          - deltachat-rpc-server-source
+          - deltachat-rpc-server-win32
+          - deltachat-rpc-server-win32-wheel
+          - deltachat-rpc-server-win64
+          - deltachat-rpc-server-win64-wheel
+          - deltachat-rpc-server-x86_64-linux
+          - deltachat-rpc-server-x86_64-linux-wheel
+          - docs
+          - libdeltachat
+          - python-docs
+
+          # Fails to build
+          #- deltachat-repl-x86_64-android
+          #- deltachat-repl-x86-android
+          #- deltachat-rpc-server-x86_64-android
+          #- deltachat-rpc-server-x86-android
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build .#${{ matrix.installable }}
+
+  build-macos:
+    name: nix build on macOS
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        installable:
+          - deltachat-rpc-server-aarch64-darwin
+
+          # Fails to bulid
+          # - deltachat-rpc-server-x86_64-darwin
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build .#${{ matrix.installable }}

CHANGELOG.md

@@ -1,5 +1,542 @@
 # Changelog
 
+## [1.155.4] - 2025-02-10
+
+### CI
+
+- Upgrade Rust from 1.84.0 to 1.84.1.
+
+### Fixes
+
+- Use CRLF newlines in vCards.
+- Make vCard parsing more robust in case of trailing newlines.
+- Do not include CRLF before MIME boundary in the part body.
+- Accept QR codes with 'broken' JSON ([#6528](https://github.com/deltachat/deltachat-core-rust/pull/6528)).
+
+### Other
+
+- Add `MessageQuote.chat_id`.
+
+### Refactor
+
+- Move even more tests into their own files ([#6521](https://github.com/deltachat/deltachat-core-rust/pull/6521)).
+
+## [1.155.3] - 2025-02-05
+
+### Fixes
+
+- Store device token in IMAP METADATA on each connection.
+
+### Miscellaneous Tasks
+
+- Upgrade iroh from 0.30 to 0.32.
+- Update `pgp` to 0.15.
+- cargo: Bump thiserror from 1.0.69 to 2.0.9.
+- cargo: Bump pin-project from 1.1.7 to 1.1.8.
+- cargo: Bump dirs from 5.0.1 to 6.0.0.
+- cargo: Bump hyper from 1.5.2 to 1.6.0.
+- cargo: Bump webpki-roots from 0.26.7 to 0.26.8.
+- cargo: Bump futures-lite from 2.5.0 to 2.6.0.
+- Update OpenSSL to fix RUSTSEC-2025-0004.
+- cargo: Bump tokio from 1.42.0 to 1.43.0.
+- cargo: Bump syn from 2.0.94 to 2.0.98.
+- cargo: Bump rustls from 0.23.20 to 0.23.22.
+- cargo: Bump data-encoding from 2.6.0 to 2.7.0.
+- cargo: Bump serde_json from 1.0.134 to 1.0.138.
+- cargo: Bump uuid from 1.11.0 to 1.12.1.
+- cargo: Bump log from 0.4.22 to 0.4.25.
+- cargo: Bump rustls-pki-types from 1.10.1 to 1.11.0.
+- Update futures-concurrency.
+
+### Documentation
+
+- Assign docs to correct object.
+
+### Tests
+
+- Make sure DCBACKUP2 compatibility does not break again.
+
+## [1.155.2] - 2025-01-31
+
+This release accidentally broke compatibility
+with previous versions of `DCBACKUP2` QR codes
+due to iroh upgrade.
+
+### API-Changes
+
+- Add `IncomingReaction.chat_id` ([#6459](https://github.com/deltachat/deltachat-core-rust/pull/6459)).
+
+### Features / Changes
+
+- Deduplicate blob files in `chat.rs`, `config.rs`, and `integration.rs`.
+- Improve logging around IMAP IDLE.
+- Upgrade to iroh@0.30.0.
+
+### Fixes
+
+- Don't remove file extension when recoding avatars.
+- Use `BufReader` when reading .xdc files.
+- No implicit member changes when we are added to the group ([#6493](https://github.com/deltachat/deltachat-core-rust/pull/6493)).
+
+### Documentation
+
+- jsonrpc: Update documentation for `select_account` and `get_selected_account_id` ([#6483](https://github.com/deltachat/deltachat-core-rust/pull/6483)).
+- jsonrpc: Add docs for some functions.
+
+## [1.155.1] - 2025-01-25
+
+### Features / Changes
+
+- Only accept SetContacts sync messages for broadcast lists.
+
+### Fixes
+
+- Don't create tombstones when synchronizing broadcast list members.
+- Use non-empty `To:` field for "saved messages".
+- Only send Chat-Group-Member-Timestamps in groups.
+- Use 0 timestamps if Chat-Group-Member-Timestamps is not set.
+
+### Refactor
+
+- Remove BlobObject::create(), use create_and_deduplicate_from_bytes() instead ([#6467](https://github.com/deltachat/deltachat-core-rust/pull/6467)).
+- Move more tests into their own files ([#6473](https://github.com/deltachat/deltachat-core-rust/pull/6473)).
+
+## [1.155.0] - 2025-01-23
+
+### API-Changes
+
+- Add JSON-RPC API to get past members.
+
+### Build system
+
+- Update Rust.
+- Increase MSRV to 1.81.0
+
+### Features / Changes
+
+- feat: Set BccSelf to true when receiving a sync message  ([#6434](https://github.com/deltachat/deltachat-core-rust/pull/6434))
+- File deduplication ([#6332](https://github.com/deltachat/deltachat-core-rust/pull/6332))
+
+### Refactor
+
+- Move tests to their own files.
+- Extract `group_changes_msgs()` function ([#6460](https://github.com/deltachat/deltachat-core-rust/pull/6460)).
+
+## [1.154.3] - 2025-01-20
+
+### Build system
+
+- Remove encoded-words from flake.nix.
+- nix: Update rust-email hash in flake.nix.
+
+### Miscellaneous Tasks
+
+- Remove unused function delete_files_in_dir() ([#6454](https://github.com/deltachat/deltachat-core-rust/pull/6454)).
+
+## [1.154.2] - 2025-01-20
+
+### Features / Changes
+
+- Add API to save messages ([#5606](https://github.com/deltachat/deltachat-core-rust/pull/5606)).
+
+### Fixes
+
+- fix: Don't accidentally remove Self from groups ([#6455](https://github.com/deltachat/deltachat-core-rust/pull/6455)).
+- Do not create tombstones for members removed from unpromoted groups.
+
+### Build system
+
+- Switch to non-git version of encoded-words.
+
+### Refactor
+
+- Make memberlist update logic easier to follow.
+
+## [1.154.1] - 2025-01-15
+
+### Tests
+
+- Expect trashing of no-op "member added" in non_member_cannot_modify_member_list.
+
+## [1.154.0] - 2025-01-15
+
+### Features / Changes
+
+- New group consistency algorithm.
+
+### Fixes
+
+- Migration: Set bcc_self=1 if it's unset and delete_server_after!=1 ([#6432](https://github.com/deltachat/deltachat-core-rust/pull/6432)).
+- Clear the config cache after every migration ([#6438](https://github.com/deltachat/deltachat-core-rust/pull/6438)).
+
+### Build system
+
+- Increase minimum supported Python version to 3.8.
+- [**breaking**] Remove jsonrpc feature flag.
+
+### CI
+
+- Update Rust to 1.84.0.
+
+### Miscellaneous Tasks
+
+- Beta Clippy suggestions ([#6422](https://github.com/deltachat/deltachat-core-rust/pull/6422)).
+
+### Refactor
+
+- Use let..else.
+- Add why_cant_send_ex() capable to only ignore specified conditions.
+- Remove unnecessary is_contact_in_chat check.
+- Eliminate remaining repeat_vars() calls ([#6359](https://github.com/deltachat/deltachat-core-rust/pull/6359)).
+
+### Tests
+
+- Use assert_eq! to compare chatlist length.
+
+## [1.153.0] - 2025-01-05
+
+### Features / Changes
+
+- Remove "jobs" from imap_markseen if folder doesn't exist ([#5870](https://github.com/deltachat/deltachat-core-rust/pull/5870)).
+- Delete `vg-request-with-auth` from IMAP after processing ([#6208](https://github.com/deltachat/deltachat-core-rust/pull/6208)).
+
+### API-Changes
+
+- Add `IncomingWebxdcNotify.chat_id` ([#6356](https://github.com/deltachat/deltachat-core-rust/pull/6356)).
+- rpc-client: Add INCOMING_REACTION to const.EventType ([#6349](https://github.com/deltachat/deltachat-core-rust/pull/6349)).
+
+### Documentation
+
+- Viewtype::Sticker may be changed to Image and how to disable that ([#6352](https://github.com/deltachat/deltachat-core-rust/pull/6352)).
+
+### Fixes
+
+- Never change Viewtype::Sticker to Image if file has non-image extension ([#6352](https://github.com/deltachat/deltachat-core-rust/pull/6352)).
+- Change BccSelf default to 0 for chatmail ([#6340](https://github.com/deltachat/deltachat-core-rust/pull/6340)).
+- Mark holiday notice messages as bot-generated.
+- Don't treat location-only and sync messages as bot ones ([#6357](https://github.com/deltachat/deltachat-core-rust/pull/6357)).
+- Update shadowsocks crate to 1.22.0 to avoid panic when parsing some QR codes.
+- Prefer to encrypt if E2eeEnabled even if peers have EncryptPreference::NoPreference.
+- Prioritize mailing list over self-sent messages.
+- Allow empty `To` field for self-sent messages.
+- Default `to_id` to self instead of 0.
+
+### Refactor
+
+- Remove unused parameter and return value from `build_body_file(…)` ([#6369](https://github.com/deltachat/deltachat-core-rust/pull/6369)).
+- Deprecate Param::ErroneousE2ee.
+- Add `emit_msgs_changed_without_msg_id`.
+- Add_parts: Remove excessive `is_mdn` checks.
+- Simplify `self_sent` condition.
+- Don't ignore get_for_contact errors.
+
+### Tests
+
+- Messages without recipients are assigned to self chat.
+- Message with empty To: field should have a valid to_id.
+- Fix `test_logged_ac_process_ffi_failure` flakiness.
+
+## [1.152.2] - 2024-12-24
+
+### Features / Changes
+
+- Emit ImexProgress(1) after receiving backup size.
+- `delete_msgs`: Use `transaction()` instead of `call_write()`.
+- Start ephemeral timers when the chat is noticed.
+- Start ephemeral timers when the chat is archived.
+- Revalidate HTTP cache entries once per minute maximum.
+
+### Fixes
+
+- Reduce number of `repeat_vars()` calls.
+- `sanitise_name`: Don't consider punctuation and control chars as part of file extension ([#6362](https://github.com/deltachat/deltachat-core-rust/pull/6362)).
+
+### Refactor
+
+- Remove marknoticed_chat_if_older_than().
+
+### Miscellaneous Tasks
+
+- Remove contrib/ directory.
+
+## [1.152.1] - 2024-12-17
+
+### Build system
+
+- Downgrade Rust version used to build binaries.
+- Reduce MSRV to 1.77.0.
+
+## [1.152.0] - 2024-12-12
+
+### API-Changes
+
+- [**breaking**] Remove `dc_prepare_msg` and `dc_msg_is_increation`.
+
+### Build system
+
+- Increase MSRV to 1.81.0.
+
+### Features / Changes
+
+- Cache HTTP GET requests.
+- Prefix server-url in info.
+- Set `mime_modified` for the last message part, not the first ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)).
+
+### Fixes
+
+- Render "message" parts in multipart messages' HTML ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)).
+- Ignore garbage at the end of the keys.
+
+## [1.151.6] - 2024-12-11
+
+### Features / Changes
+
+- Don't add "Failed to send message to ..." info messages to group chats.
+- Add info messages about implicit membership changes if group member list is recreated ([#6314](https://github.com/deltachat/deltachat-core-rust/pull/6314)).
+
+### Fixes
+
+- Add self-addition message to chat when recreating member list.
+- Do not subscribe to heartbeat if already subscribed via metadata.
+
+### Build system
+
+- Add idna 0.5.0 exception into deny.toml.
+
+### Documentation
+
+- Update links to Node.js bindings in the README.
+
+### Refactor
+
+- Factor out `wait_for_all_work_done()`.
+
+### Tests
+
+- Notifiy more prominently & in more tests about false positives when running `cargo test` ([#6308](https://github.com/deltachat/deltachat-core-rust/pull/6308)).
+
+## [1.151.5] - 2024-12-05
+
+### API-Changes
+
+- [**breaking**] Remove dc_all_work_done().
+
+### Security
+
+- cargo: Update rPGP to 0.14.2.
+
+  This fixes [Panics on Malformed Untrusted Input](https://github.com/rpgp/rpgp/security/advisories/GHSA-9rmp-2568-59rv)
+  and [Potential Resource Exhaustion when handling Untrusted Messages](https://github.com/rpgp/rpgp/security/advisories/GHSA-4grw-m28r-q285).
+  This allows the attacker to crash the application via specially crafted messages and keys.
+  We recommend all users and bot operators to upgrade to the latest version.
+  There is no impact on the confidentiality of the messages and keys so no action other than upgrading is needed.
+
+### Fixes
+
+- Store plaintext in mime_headers of truncated sent messages ([#6273](https://github.com/deltachat/deltachat-core-rust/pull/6273)).
+
+### Documentation
+
+- Document `push` module.
+- Remove mention of non-existent `nightly` feature.
+
+### Tests
+
+- Fix panic in `receive_emails` benchmark ([#6306](https://github.com/deltachat/deltachat-core-rust/pull/6306)).
+
+## [1.151.4] - 2024-12-03
+
+### Features / Changes
+
+- Encrypt notification tokens.
+
+### Fixes
+
+- Replace connectivity state "Connected" with "Preparing".
+
+### Miscellaneous Tasks
+
+- Beta clippy suggestions ([#6271](https://github.com/deltachat/deltachat-core-rust/pull/6271)).
+
+### Tests
+
+- Fix `cargo check` for `receive_emails` benchmark.
+
+### CI
+
+- Also run cargo check without all-features.
+
+## [1.151.3] - 2024-12-02
+
+### API-Changes
+
+- Remove experimental `request_internet_access` option from webxdc's `manifest.toml`.
+- Add getWebxdcHref to json api ([#6281](https://github.com/deltachat/deltachat-core-rust/pull/6281)).
+
+### CI
+
+- Update Rust to 1.83.0.
+
+### Documentation
+
+- Update dc_msg_get_info_type() and dc_get_securejoin_qr() ([#6269](https://github.com/deltachat/deltachat-core-rust/pull/6269)).
+- Fix references to iroh-related headers in peer_channels docs.
+- Improve CFFI docs, link to corresponding JSON-RPC docs.
+
+### Features / Changes
+
+- Allow the user to replace maps integration ([#5678](https://github.com/deltachat/deltachat-core-rust/pull/5678)).
+- Mark saved messages chat as protected.
+
+### Fixes
+
+- Close iroh endpoint when I/O is stopped.
+- Do not add protection messages to Saved Messages chat.
+- Mark Saved Messages chat as protected if it exists.
+- Sync chat action even if sync message arrives before first one from contact ([#6259](https://github.com/deltachat/deltachat-core-rust/pull/6259)).
+
+### Refactor
+
+- Remove some .unwrap() calls.
+- Create_status_update_record: Remove double check of info_msg_id.
+- Use Option::or_else() to dedup emitting IncomingWebxdcNotify.
+
+## [1.151.2] - 2024-11-26
+
+### API-Changes
+
+- Deprecate webxdc `descr` parameter ([#6255](https://github.com/deltachat/deltachat-core-rust/pull/6255)).
+
+### Features / Changes
+
+- AEAP: Check that the old peerstate verified key fingerprint hasn't changed when removing it.
+- Add `AccountsChanged` and `AccountsItemChanged` events ([#6118](https://github.com/deltachat/deltachat-core-rust/pull/6118)).
+- Do not use format=flowed in outgoing messages ([#6256](https://github.com/deltachat/deltachat-core-rust/pull/6256)).
+- Add webxdc limits api.
+- Add href to IncomingWebxdcNotify event ([#6266](https://github.com/deltachat/deltachat-core-rust/pull/6266)).
+
+### Fixes
+
+- Revert treating some transient SMTP errors as permanent.
+
+### Refactor
+
+- Create_status_update_record: Get rid of `notify` var.
+
+### Tests
+
+- Check that IncomingMsg isn't emitted for reactions.
+
+## [1.151.1] - 2024-11-24
+
+### Build system
+
+- nix: Fix deltachat-rpc-server-source installable.
+
+### CI
+
+- Test building nix targets to avoid regressions.
+
+## [1.151.0] - 2024-11-23
+
+### Features / Changes
+
+- Trim whitespace from scanned QR codes.
+- Use privacy-preserving webxdc addresses ([#6237](https://github.com/deltachat/deltachat-core-rust/pull/6237)).
+- Webxdc notify ([#6230](https://github.com/deltachat/deltachat-core-rust/pull/6230)).
+- `update.href` api ([#6248](https://github.com/deltachat/deltachat-core-rust/pull/6248)).
+
+### Fixes
+
+- Never notify SELF ([#6251](https://github.com/deltachat/deltachat-core-rust/pull/6251)).
+
+### Build system
+
+- Use underscores in deltachat-rpc-server source package filename.
+- Remove imap_tools from dependencies ([#6238](https://github.com/deltachat/deltachat-core-rust/pull/6238)).
+- cargo: Update Rustls from 0.23.14 to 0.23.18.
+- deps: Bump curve25519-dalek from 3.2.0 to 4.1.3 in /fuzz.
+
+### Documentation
+
+- Move style guide into a separate document.
+- Clarify DC_EVENT_INCOMING_WEBXDC_NOTIFY documentation ([#6249](https://github.com/deltachat/deltachat-core-rust/pull/6249)).
+
+### Tests
+
+- After AEAP, 1:1 chat isn't available for sending, but unprotected groups are ([#6222](https://github.com/deltachat/deltachat-core-rust/pull/6222)).
+
+## [1.150.0] - 2024-11-21
+
+### API-Changes
+
+- Correct `DC_CERTCK_ACCEPT_*` values and docs ([#6176](https://github.com/deltachat/deltachat-core-rust/pull/6176)).
+
+### Features / Changes
+
+- Use Rustls for connections with strict TLS ([#6186](https://github.com/deltachat/deltachat-core-rust/pull/6186)).
+- Experimental header protection for Autocrypt.
+- Tune down io-not-started info in connectivity-html.
+- Clear config cache in start_io() ([#6228](https://github.com/deltachat/deltachat-core-rust/pull/6228)).
+- Line-before-quote may be up to 120 character long instead of 80.
+- Use i.delta.chat in qr codes ([#6223](https://github.com/deltachat/deltachat-core-rust/pull/6223)).
+
+### Fixes
+
+- Prevent accidental wrong-password-notifications ([#6122](https://github.com/deltachat/deltachat-core-rust/pull/6122)).
+- Remove footers from "Show Full Message...".
+- `send_msg_to_smtp`: Return Ok if `smtp` row is deleted in parallel.
+- Only add "member added/removed" messages if they actually do that ([#5992](https://github.com/deltachat/deltachat-core-rust/pull/5992)).
+- Do not fail to load chatlist summary if the message got removed.
+- deltachat-jsonrpc: Do not fail `get_chatlist_items_by_entries` if the message got deleted.
+- deltachat-jsonrpc: Do not fail `get_draft` if draft is deleted.
+- `markseen_msgs`: Limit not yet downloaded messages state to `InNoticed` ([#2970](https://github.com/deltachat/deltachat-core-rust/pull/2970)).
+- Update state of message when fully downloading it.
+- Dont overwrite equal drafts ([#6212](https://github.com/deltachat/deltachat-core-rust/pull/6212)).
+
+### Build system
+
+- Silence RUSTSEC-2024-0384.
+- cargo: Update rPGP from 0.13.2 to 0.14.0.
+- cargo: Update futures-concurrency from 7.6.1 to 7.6.2.
+- Update flake.nix ([#6200](https://github.com/deltachat/deltachat-core-rust/pull/6200))
+
+### CI
+
+- Ensure flake is formatted.
+
+### Documentation
+
+- Scanned proxies are added and normalized.
+
+### Refactor
+
+- Fix nightly clippy warnings.
+- Remove slicing from `is_file_in_use`.
+- Remove unnecessary `allow(clippy::indexing_slicing)`.
+- Don't use slicing in `remove_nonstandard_footer`.
+- Do not use slicing in `qr` module.
+- Eliminate indexing in `compute_mailinglist_name`.
+- Remove unused `allow(clippy::indexing_slicing)`.
+- Remove indexing/slicing from `remove_message_footer`.
+- Remove indexing/slicing from `squash_attachment_parts`.
+- Remove unused allow(clippy::indexing_slicing) for heuristically_parse_ndn.
+- Remove indexing/slicing from `parse_message_ids`.
+- Remove slicing from `remove_bottom_quote`.
+- Get rid of slicing in `remove_top_quote`.
+- Remove unused allow(clippy::indexing_slicing) from 'truncate'.
+- Forbid clippy::indexing_slicing.
+- Forbid clippy::string_slice.
+- Delete chat in a transaction.
+- Fix typo in `context.rs`.
+
+### Tests
+
+- Remove all calls to print() from deltachat-rpc-client tests.
+- Reply to protected group from MUA.
+- Mark not downloaded message as seen ([#2970](https://github.com/deltachat/deltachat-core-rust/pull/2970)).
+- Mark `receive_imf()` as only for tests and "internals" feature ([#6235](https://github.com/deltachat/deltachat-core-rust/pull/6235)).
+
 ## [1.149.0] - 2024-11-05
 
 ### Build system
@@ -236,7 +773,7 @@ This reverts commit 6f22ce2722b51773d7fbb0d89e4764f963cafd91..
 - Reset quota on configured address change ([#5908](https://github.com/deltachat/deltachat-core-rust/pull/5908)).
 - Do not emit progress 1000 when configuration is cancelled.
 - Assume file extensions are 32 chars max and don't contain whitespace ([#5338](https://github.com/deltachat/deltachat-core-rust/pull/5338)).
-- Readd tokens.foreign_id column ([#6038](https://github.com/deltachat/deltachat-core-rust/pull/6038)).
+- Re-add tokens.foreign_id column ([#6038](https://github.com/deltachat/deltachat-core-rust/pull/6038)).
 
 ### Miscellaneous Tasks
 
@@ -827,7 +1364,7 @@ This reverts commit 6f22ce2722b51773d7fbb0d89e4764f963cafd91..
 
 ### Tests
 
-- deltachat-rpc-client: reenable `log_cli`.
+- deltachat-rpc-client: re-enable `log_cli`.
 
 ## [1.140.0] - 2024-06-04
 
@@ -1764,7 +2301,7 @@ This reverts commit 6f22ce2722b51773d7fbb0d89e4764f963cafd91..
   - Mark 1:1 chat as verified for Bob early. 1:1 chat with Alice is verified as soon as Alice's key is verified rather than at the end of the protocol.
 - Put Message-ID into hidden headers and take it from there on receiver ([#4798](https://github.com/deltachat/deltachat-core-rust/pull/4798)). This works around servers which generate their own Message-ID and overwrite the one generated by Delta Chat.
 - deltachat-repl: Enable INFO logging by default and add timestamps.
-- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elments based on the configuration key which is a part of the event.
+- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elements based on the configuration key which is a part of the event.
 - Sync contact creation/rename across devices ([#5163](https://github.com/deltachat/deltachat-core-rust/pull/5163)).
 - Encrypt MDNs ([#5175](https://github.com/deltachat/deltachat-core-rust/pull/5175)).
 - Only try to configure non-strict TLS checks if explicitly set ([#5181](https://github.com/deltachat/deltachat-core-rust/pull/5181)).
@@ -4481,14 +5018,10 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 - new qr-code type `DC_QR_WEBRTC` #1779
 
-- new `dc_chatlist_get_summary2()` api #1771
-
 - tweak smtp-timeout for larger mails #1782
 
 - optimize read-receipts #1765
 
-- Allow http scheme for DCACCOUNT URLs #1770
-
 - improve tests #1769
 
 - bug fixes #1766 #1772 #1773 #1775 #1776 #1777
@@ -5233,3 +5766,24 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.148.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.5..v1.148.6
 [1.148.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.6..v1.148.7
 [1.149.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.7..v1.149.0
+[1.150.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.149.0..v1.150.0
+[1.151.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.150.0..v1.151.0
+[1.151.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.0..v1.151.1
+[1.151.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.1..v1.151.2
+[1.151.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.2..v1.151.3
+[1.151.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.3..v1.151.4
+[1.151.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.4..v1.151.5
+[1.151.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.5..v1.151.6
+[1.152.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.6..v1.152.0
+[1.152.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.152.0..v1.152.1
+[1.152.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.152.1..v1.152.2
+[1.153.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.152.2..v1.153.0
+[1.154.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.153.0..v1.154.0
+[1.154.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.154.0..v1.154.1
+[1.154.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.154.1..v1.154.2
+[1.154.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.154.2..v1.154.3
+[1.155.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.154.3..v1.155.0
+[1.155.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.0..v1.155.1
+[1.155.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.1..v1.155.2
+[1.155.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.2..v1.155.3
+[1.155.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.155.3..v1.155.4

CMakeLists.txt

@@ -27,7 +27,7 @@ add_custom_command(
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release --features jsonrpc
+	${CARGO} build --target-dir=${CMAKE_BINARY_DIR}/target --release
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deltachat-ffi
 )
 

CONTRIBUTING.md

@@ -1,6 +1,6 @@
-# Contributing guidelines
+# Contributing to Delta Chat
 
-## Reporting bugs
+## Bug reports
 
 If you found a bug, [report it on GitHub](https://github.com/deltachat/deltachat-core-rust/issues).
 If the bug you found is specific to
@@ -9,178 +9,114 @@ If the bug you found is specific to
 [Desktop](https://github.com/deltachat/deltachat-desktop/issues),
 report it to the corresponding repository.
 
-## Proposing features
+## Feature proposals
 
 If you have a feature request, create a new topic on the [forum](https://support.delta.chat/).
 
-## Contributing code
+## Code contributions
 
-If you want to contribute a code, [open a Pull Request](https://github.com/deltachat/deltachat-core-rust/pulls).
+If you want to contribute a code, follow this guide.
 
-If you have write access to the repository,
-push a branch named `<username>/<feature>`
-so it is clear who is responsible for the branch,
-and open a PR proposing to merge the change.
-Otherwise fork the repository and create a branch in your fork.
+1. **Select an issue to work on.**
+
+   If you have an write access to the repository, assign the issue to yourself.
+   Otherwise state in the comment that you are going to work on the issue
+   to avoid duplicate work.
+
+   If the issue does not exist yet, create it first.
+
+2. **Write the code.**
+
+   Follow the [coding conventions](STYLE.md) when writing the code.
+
+3. **Commit the code.**
+
+   If you have write access to the repository,
+   push a branch named `<username>/<feature>`
+   so it is clear who is responsible for the branch,
+   and open a PR proposing to merge the change.
+   Otherwise fork the repository and create a branch in your fork.
+
+   Commit messages follow the [Conventional Commits] notation.
+   We use [git-cliff] to generate the changelog from commit messages before the release.
+
+   With **`git cliff --unreleased`**, you can check how the changelog entry for your commit will look.
+
+   The following prefix types are used:
+   - `feat`: Features, e.g. "feat: Pause IO for BackupProvider". If you are unsure what's the category of your commit, you can often just use `feat`.
+   - `fix`: Bug fixes, e.g. "fix: delete `smtp` rows when message sending is cancelled"
+   - `api`: API changes, e.g. "api(rust): add `get_msg_read_receipts(context, msg_id)`"
+   - `refactor`: Refactorings, e.g. "refactor: iterate over `msg_ids` without `.iter()`"
+   - `perf`: Performance improvements, e.g. "perf: improve SQLite performance with `PRAGMA synchronous=normal`"
+   - `test`: Test changes and improvements to the testing framework.
+   - `build`: Build system and tool configuration changes, e.g. "build(git-cliff): put "ci" commits into "CI" section of changelog"
+   - `ci`: CI configuration changes, e.g. "ci: limit artifact retention time for `libdeltachat.a` to 1 day"
+   - `docs`: Documentation changes, e.g. "docs: add contributing guidelines"
+   - `chore`: miscellaneous tasks, e.g. "chore: add `.DS_Store` to `.gitignore`"
+
+   Release preparation commits are marked as "chore(release): prepare for X.Y.Z"
+   as described in [releasing guide](RELEASE.md).
+
+   Use a `!` to mark breaking changes, e.g. "api!: Remove `dc_chat_can_send`".
+
+   Alternatively, breaking changes can go into the commit description, e.g.:
+
+   ```
+   fix: Fix race condition and db corruption when a message was received during backup
+
+   BREAKING CHANGE: You have to call `dc_stop_io()`/`dc_start_io()` before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`
+   ```
+
+4. [**Open a Pull Request**](https://github.com/deltachat/deltachat-core-rust/pulls).
+
+   Refer to the corresponding issue.
+
+   If you intend to squash merge the PR from the web interface,
+   make sure the PR title follows the conventional commits notation
+   as it will end up being a commit title.
+   Otherwise make sure each commit title follows the conventional commit notation.
+
+5. **Make sure all CI checks succeed.**
+
+   CI runs the tests and checks code formatting.
+
+   While it is running, self-review your PR to make sure all the changes you expect are there
+   and there are no accidentally committed unrelated changes and files.
+
+   Push the necessary fixup commits or force-push to your branch if needed.
+
+6. **Ask for review.**
+
+   Use built-in GitHub feature to request a review from suggested reviewers.
+
+   If you do not have write access to the repository, ask for review in the comments.
+
+7. **Merge the PR.**
+
+   Once a PR has an approval and passes CI, it can be merged.
+
+   PRs from a branch created in the main repository,
+   i.e. authored by those who have write access, are merged by their authors.
+
+   This is to ensure that PRs are merged as intended by the author,
+   e.g. as a squash merge, by rebasing from the web interface or manually from the command line.
+
+   If you have multiple changes in one PR, do a rebase merge.
+   Otherwise, you should usually do a squash merge.
+
+   If PR author does not have write access to the repository,
+   maintainers who reviewed the PR can merge it.
+
+   If you do not have access to the repository and created a PR from a fork,
+   ask the maintainers to merge the PR and say how it should be merged.
+
+## Other ways to contribute
+
+For other ways to contribute, refer to the [website](https://delta.chat/en/contribute).
 
 You can find the list of good first issues
 and a link to this guide
 on the contributing page: <https://github.com/deltachat/deltachat-core-rust/contribute>
 
-### Coding conventions
-
-We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
-Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
-
-### SQL
-
-Multi-line SQL statements should be formatted using string literals,
-for example
-```
-    sql.execute(
-        "CREATE TABLE messages (
-id INTEGER PRIMARY KEY AUTOINCREMENT,
-text TEXT DEFAULT '' NOT NULL -- message text
-) STRICT",
-    )
-    .await?;
-```
-
-Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
-or [`indoc!](https://docs.rs/indoc).
-Do not escape newlines like this:
-```
-    sql.execute(
-        "CREATE TABLE messages ( \
-id INTEGER PRIMARY KEY AUTOINCREMENT, \
-text TEXT DEFAULT '' NOT NULL \
-) STRICT",
-    )
-    .await?;
-```
-Escaping newlines
-is prone to errors like this if space before backslash is missing:
-```
-"SELECT foo\
- FROM bar"
-```
-Literal above results in `SELECT fooFROM bar` string.
-This style also does not allow using `--` comments.
-
----
-
-Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
-to make SQLite check column types.
-
-Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
-This avoids reuse of the row IDs and can avoid dangerous bugs
-like forwarding wrong message because the message was deleted
-and another message took its row ID.
-
-Declare all new columns as `NOT NULL`
-and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
-Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
-If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
-Use `HAVING COUNT(*) > 0` clause
-to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
-
-Don't delete unused columns too early, but maybe after several months/releases, unused columns are
-still used by older versions, so deleting them breaks downgrading the core or importing a backup in
-an older version. Also don't change the column type, consider adding a new column with another name
-instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
-keyword doesn't help here.
-
-### Commit messages
-
-Commit messages follow the [Conventional Commits] notation.
-We use [git-cliff] to generate the changelog from commit messages before the release.
-
-With **`git cliff --unreleased`**, you can check how the changelog entry for your commit will look.
-
-The following prefix types are used:
-- `feat`: Features, e.g. "feat: Pause IO for BackupProvider". If you are unsure what's the category of your commit, you can often just use `feat`.
-- `fix`: Bug fixes, e.g. "fix: delete `smtp` rows when message sending is cancelled"
-- `api`: API changes, e.g. "api(rust): add `get_msg_read_receipts(context, msg_id)`"
-- `refactor`: Refactorings, e.g. "refactor: iterate over `msg_ids` without `.iter()`"
-- `perf`: Performance improvements, e.g. "perf: improve SQLite performance with `PRAGMA synchronous=normal`"
-- `test`: Test changes and improvements to the testing framework.
-- `build`: Build system and tool configuration changes, e.g. "build(git-cliff): put "ci" commits into "CI" section of changelog"
-- `ci`: CI configuration changes, e.g. "ci: limit artifact retention time for `libdeltachat.a` to 1 day"
-- `docs`: Documentation changes, e.g. "docs: add contributing guidelines"
-- `chore`: miscellaneous tasks, e.g. "chore: add `.DS_Store` to `.gitignore`"
-
-Release preparation commits are marked as "chore(release): prepare for vX.Y.Z".
-
-If you intend to squash merge the PR from the web interface,
-make sure the PR title follows the conventional commits notation
-as it will end up being a commit title.
-Otherwise make sure each commit title follows the conventional commit notation.
-
-#### Breaking Changes
-
-Use a `!` to mark breaking changes, e.g. "api!: Remove `dc_chat_can_send`".
-
-Alternatively, breaking changes can go into the commit description, e.g.:
-
-```
-fix: Fix race condition and db corruption when a message was received during backup
-
-BREAKING CHANGE: You have to call `dc_stop_io()`/`dc_start_io()` before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`
-```
-
-#### Multiple Changes in one PR
-
-If you have multiple changes in one PR, create multiple conventional commits, and then do a rebase merge. Otherwise, you should usually do a squash merge.
-
-[Clippy]: https://doc.rust-lang.org/clippy/
 [Conventional Commits]: https://www.conventionalcommits.org/
 [git-cliff]: https://git-cliff.org/
-
-### Errors
-
-Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.
-When using [`Context`](https://docs.rs/anyhow/latest/anyhow/trait.Context.html),
-capitalize it but do not add a full stop as the contexts will be separated by `:`.
-For example:
-```
-.with_context(|| format!("Unable to trash message {msg_id}"))
-```
-
-All errors should be handled in one of these ways:
-- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
-- With `.log_err().ok()`.
-- Bubbled up with `?`.
-
-`backtrace` feature is enabled for `anyhow` crate
-and `debug = 1` option is set in the test profile.
-This allows to run `RUST_BACKTRACE=1 cargo test`
-and get a backtrace with line numbers in resultified tests
-which return `anyhow::Result`.
-
-### Logging
-
-For logging, use `info!`, `warn!` and `error!` macros.
-Log messages should be capitalized and have a full stop in the end. For example:
-```
-info!(context, "Ignoring addition of {added_addr:?} to {chat_id}.");
-```
-
-Format anyhow errors with `{:#}` to print all the contexts like this:
-```
-error!(context, "Failed to set selfavatar timestamp: {err:#}.");
-```
-
-### Reviewing
-
-Once a PR has an approval and passes CI, it can be merged.
-
-PRs from a branch created in the main repository, i.e. authored by those who have write access, are merged by their authors.
-This is to ensure that PRs are merged as intended by the author,
-e.g. as a squash merge, by rebasing from the web interface or manually from the command line.
-
-If you do not have access to the repository and created a PR from a fork,
-ask the maintainers to merge the PR and say how it should be merged.
-
-## Other ways to contribute
-
-For other ways to contribute, refer to the [website](https://delta.chat/en/contribute).

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.149.0"
+version = "1.155.4"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.77"
+rust-version = "1.81"
 repository = "https://github.com/deltachat/deltachat-core-rust"
 
 [profile.dev]
@@ -39,36 +39,37 @@ format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
-async-broadcast = "0.7.1"
+async-broadcast = "0.7.2"
 async-channel = { workspace = true }
 async-imap = { version = "0.10.2", default-features = false, features = ["runtime-tokio", "compress"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
-async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.10", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.17", default-features = false, features = ["deflate", "tokio-fs"] }
 base64 = { workspace = true }
 brotli = { version = "7", default-features=false, features = ["std"] }
 bytes = "1"
 chrono = { workspace = true, features = ["alloc", "clock", "std"] }
+data-encoding = "2.7.0"
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
+encoded-words = "0.2"
 escaper = "0.1"
-fast-socks5 = "0.9"
+fast-socks5 = "0.10"
 fd-lock = "4"
 futures-lite = { workspace = true }
 futures = { workspace = true }
 hex = "0.4.0"
-hickory-resolver = "=0.25.0-alpha.2"
+hickory-resolver = "=0.25.0-alpha.4"
 http-body-util = "0.1.2"
 humansize = "2"
 hyper = "1"
 hyper-util = "0.1.10"
-image = { version = "0.25.4", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh-gossip = { version = "0.28.1", default-features = false, features = ["net"] }
-iroh-net = { version = "0.28.1", default-features = false }
-kamadak-exif = "0.6.0"
+image = { version = "0.25.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+iroh-gossip = { version = "0.32", default-features = false, features = ["net"] }
+iroh = { version = "0.32", default-features = false }
+kamadak-exif = "0.6.1"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = { workspace = true }
-mailparse = "0.15"
+mailparse = "0.16"
 mime = "0.3.17"
 num_cpus = "1.16"
 num-derive = "0.4"
@@ -76,7 +77,7 @@ num-traits = { workspace = true }
 once_cell = { workspace = true }
 parking_lot = "0.12"
 percent-encoding = "2.3"
-pgp = { version = "0.13.2", default-features = false }
+pgp = { version = "0.15.0", default-features = false }
 pin-project = "1"
 qrcodegen = "1.7.0"
 quick-xml = "0.37"
@@ -85,14 +86,15 @@ rand = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustls-pki-types = "1.10.0"
-rustls = { version = "0.23.14", default-features = false }
+rustls-pki-types = "1.11.0"
+rustls = { version = "0.23.22", default-features = false }
 sanitize-filename = { workspace = true }
 serde_json = { workspace = true }
 serde_urlencoded = "0.7.1"
 serde = { workspace = true, features = ["derive"] }
 sha-1 = "0.10"
-shadowsocks = { version = "1.21.0", default-features = false, features = ["aead-cipher-2022"] }
+sha2 = "0.10"
+shadowsocks = { version = "1.22.0", default-features = false, features = ["aead-cipher", "aead-cipher-2022"] }
 smallvec = "1.13.2"
 strum = "0.26"
 strum_macros = "0.26"
@@ -100,15 +102,16 @@ tagger = "4.3.4"
 textwrap = "0.16.1"
 thiserror = { workspace = true }
 tokio-io-timeout = "1.2.0"
-tokio-rustls = { version = "0.26.0", default-features = false }
-tokio-stream = { version = "0.1.16", features = ["fs"] }
+tokio-rustls = { version = "0.26.1", default-features = false }
+tokio-stream = { version = "0.1.17", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-webpki-roots = "0.26.6"
+webpki-roots = "0.26.8"
+blake3 = "1.5.5"
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
@@ -119,7 +122,7 @@ nu-ansi-term = { workspace = true }
 pretty_assertions = "1.4.1"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = { workspace = true }
-testdir = "0.9.0"
+testdir = "0.9.3"
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 
 [workspace]
@@ -149,6 +152,7 @@ harness = false
 
 [[bench]]
 name = "receive_emails"
+required-features = ["internals"]
 harness = false
 
 [[bench]]
@@ -167,12 +171,12 @@ harness = false
 anyhow = "1"
 async-channel = "2.3.1"
 base64 = "0.22"
-chrono = { version = "0.4.38", default-features = false }
+chrono = { version = "0.4.39", default-features = false }
 deltachat-contact-tools = { path = "deltachat-contact-tools" }
 deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
 deltachat = { path = ".", default-features = false }
 futures = "0.3.31"
-futures-lite = "2.4.0"
+futures-lite = "2.6.0"
 libc = "0.2"
 log = "0.4"
 nu-ansi-term = "0.46"
@@ -184,10 +188,10 @@ rusqlite = "0.32"
 sanitize-filename = "0.5"
 serde = "1.0"
 serde_json = "1"
-tempfile = "3.13.0"
-thiserror = "1"
+tempfile = "3.14.0"
+thiserror = "2"
 tokio = "1"
-tokio-util = "0.7.11"
+tokio-util = "0.7.13"
 tracing-subscriber = "0.3"
 yerpc = "0.6.2"
 

README.md

@@ -161,7 +161,6 @@ $ cargo bolero test fuzz_format_flowed --release=false -e afl -s NONE
 ## Features
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
-- `nightly`: Enable nightly only performance and security related features.
 
 ## Update Provider Data
 
@@ -178,8 +177,8 @@ Language bindings are available for:
 
 - **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
 - **Node.js** 
-  - over cffi: \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
-  - over jsonrpc built with napi.rs (experimental): \[[📂 source](https://github.com/deltachat/napi-jsonrpc) | [📦 npm](https://www.npmjs.com/package/@deltachat/napi-jsonrpc)\]
+  - over JSON-RPC: \[[📂 source](./deltachat-rpc-client) | [📦 npm](https://www.npmjs.com/package/@deltachat/jsonrpc-client) | [📚 docs](https://js.jsonrpc.delta.chat/)\]
+  - over CFFI[^1]: \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat/)\]
 - **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
 - **Go**
   - over jsonrpc: \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]

RELEASE.md

@@ -14,8 +14,8 @@ For example, to release version 1.116.0 of the core, do the following steps.
 5. Commit the changes as `chore(release): prepare for 1.116.0`.
    Optionally, use a separate branch like `prep-1.116.0` for this commit and open a PR for review.
 
-6. Tag the release: `git tag -a v1.116.0`.
+6. Tag the release: `git tag --annotate v1.116.0`.
 
 7. Push the release tag: `git push origin v1.116.0`.
 
-8. Create a GitHub release: `gh release create v1.116.0 -n ''`.
+8. Create a GitHub release: `gh release create v1.116.0 --notes ''`.

STYLE.md

@@ -0,0 +1,98 @@
+# Coding conventions
+
+We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
+Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
+
+[Clippy]: https://doc.rust-lang.org/clippy/
+
+## SQL
+
+Multi-line SQL statements should be formatted using string literals,
+for example
+```
+    sql.execute(
+        "CREATE TABLE messages (
+id INTEGER PRIMARY KEY AUTOINCREMENT,
+text TEXT DEFAULT '' NOT NULL -- message text
+) STRICT",
+    )
+    .await?;
+```
+
+Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
+or [`indoc!](https://docs.rs/indoc).
+Do not escape newlines like this:
+```
+    sql.execute(
+        "CREATE TABLE messages ( \
+id INTEGER PRIMARY KEY AUTOINCREMENT, \
+text TEXT DEFAULT '' NOT NULL \
+) STRICT",
+    )
+    .await?;
+```
+Escaping newlines
+is prone to errors like this if space before backslash is missing:
+```
+"SELECT foo\
+ FROM bar"
+```
+Literal above results in `SELECT fooFROM bar` string.
+This style also does not allow using `--` comments.
+
+---
+
+Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
+to make SQLite check column types.
+
+Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
+This avoids reuse of the row IDs and can avoid dangerous bugs
+like forwarding wrong message because the message was deleted
+and another message took its row ID.
+
+Declare all new columns as `NOT NULL`
+and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
+Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
+If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
+Use `HAVING COUNT(*) > 0` clause
+to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
+
+Don't delete unused columns too early, but maybe after several months/releases, unused columns are
+still used by older versions, so deleting them breaks downgrading the core or importing a backup in
+an older version. Also don't change the column type, consider adding a new column with another name
+instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
+keyword doesn't help here.
+
+## Errors
+
+Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.
+When using [`Context`](https://docs.rs/anyhow/latest/anyhow/trait.Context.html),
+capitalize it but do not add a full stop as the contexts will be separated by `:`.
+For example:
+```
+.with_context(|| format!("Unable to trash message {msg_id}"))
+```
+
+All errors should be handled in one of these ways:
+- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+- With `.log_err().ok()`.
+- Bubbled up with `?`.
+
+`backtrace` feature is enabled for `anyhow` crate
+and `debug = 1` option is set in the test profile.
+This allows to run `RUST_BACKTRACE=1 cargo test`
+and get a backtrace with line numbers in resultified tests
+which return `anyhow::Result`.
+
+## Logging
+
+For logging, use `info!`, `warn!` and `error!` macros.
+Log messages should be capitalized and have a full stop in the end. For example:
+```
+info!(context, "Ignoring addition of {added_addr:?} to {chat_id}.");
+```
+
+Format anyhow errors with `{:#}` to print all the contexts like this:
+```
+error!(context, "Failed to set selfavatar timestamp: {err:#}.");
+```

benches/receive_emails.rs

@@ -12,18 +12,18 @@ use deltachat::{
 };
 use tempfile::tempdir;
 
-async fn recv_all_emails(context: Context) -> Context {
+async fn recv_all_emails(context: Context, iteration: u32) -> Context {
     for i in 0..100 {
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Mr.{iteration}.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com
 From: sender@testrun.org
 Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
-In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Mr.{iteration}.{i_dec}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
@@ -41,11 +41,11 @@ Hello {i}",
 
 /// Receive 100 emails that remove charlie@example.com and add
 /// him back
-async fn recv_groupmembership_emails(context: Context) -> Context {
+async fn recv_groupmembership_emails(context: Context, iteration: u32) -> Context {
     for i in 0..50 {
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Gr.{iteration}.ADD.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
 From: sender@testrun.org
@@ -53,13 +53,12 @@ Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
 Chat-Group-Member-Added: charlie@example.com
-In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Gr.{iteration}.REMOVE.{i_dec}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 
 Hello {i}",
-            i = i,
             i_dec = i - 1,
         );
         receive_imf(&context, black_box(imf_raw.as_bytes()), false)
@@ -68,7 +67,7 @@ Hello {i}",
 
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Gr.{iteration}.REMOVE.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
 From: sender@testrun.org
@@ -76,14 +75,12 @@ Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
 Chat-Group-Member-Removed: charlie@example.com
-In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Gr.{iteration}.ADD.{i}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 
-Hello {i}",
-            i = i,
-            i_dec = i - 1,
+Hello {i}"
         );
         receive_imf(&context, black_box(imf_raw.as_bytes()), false)
             .await
@@ -129,11 +126,13 @@ fn criterion_benchmark(c: &mut Criterion) {
     group.bench_function("Receive 100 simple text msgs", |b| {
         let rt = tokio::runtime::Runtime::new().unwrap();
         let context = rt.block_on(create_context());
+        let mut i = 0;
 
         b.to_async(&rt).iter(|| {
             let ctx = context.clone();
+            i += 1;
             async move {
-                recv_all_emails(black_box(ctx)).await;
+                recv_all_emails(black_box(ctx), i).await;
             }
         });
     });
@@ -142,11 +141,13 @@ fn criterion_benchmark(c: &mut Criterion) {
         |b| {
             let rt = tokio::runtime::Runtime::new().unwrap();
             let context = rt.block_on(create_context());
+            let mut i = 0;
 
             b.to_async(&rt).iter(|| {
                 let ctx = context.clone();
+                i += 1;
                 async move {
-                    recv_groupmembership_emails(black_box(ctx)).await;
+                    recv_groupmembership_emails(black_box(ctx), i).await;
                 }
             });
         },

contrib/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

deltachat-contact-tools/src/lib.rs

@@ -15,7 +15,8 @@
     clippy::explicit_into_iter_loop,
     clippy::cloned_instead_of_copied
 )]
-#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 #![allow(
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,
@@ -77,21 +78,21 @@ pub fn make_vcard(contacts: &[VcardContact]) -> String {
         let addr = &c.addr;
         let display_name = c.display_name();
         res += &format!(
-            "BEGIN:VCARD\n\
-             VERSION:4.0\n\
-             EMAIL:{addr}\n\
-             FN:{display_name}\n"
+            "BEGIN:VCARD\r\n\
+             VERSION:4.0\r\n\
+             EMAIL:{addr}\r\n\
+             FN:{display_name}\r\n"
         );
         if let Some(key) = &c.key {
-            res += &format!("KEY:data:application/pgp-keys;base64,{key}\n");
+            res += &format!("KEY:data:application/pgp-keys;base64,{key}\r\n");
         }
         if let Some(profile_image) = &c.profile_image {
-            res += &format!("PHOTO:data:image/jpeg;base64,{profile_image}\n");
+            res += &format!("PHOTO:data:image/jpeg;base64,{profile_image}\r\n");
         }
         if let Some(timestamp) = format_timestamp(c) {
-            res += &format!("REV:{timestamp}\n");
+            res += &format!("REV:{timestamp}\r\n");
         }
-        res += "END:VCARD\n";
+        res += "END:VCARD\r\n";
     }
     res
 }
@@ -205,22 +206,21 @@ pub fn parse_vcard(vcard: &str) -> Vec<VcardContact> {
             } else if let Some(rev) = vcard_property(line, "rev") {
                 datetime.get_or_insert(rev);
             } else if line.eq_ignore_ascii_case("END:VCARD") {
+                let (authname, addr) =
+                    sanitize_name_and_addr(display_name.unwrap_or(""), addr.unwrap_or(""));
+
+                contacts.push(VcardContact {
+                    authname,
+                    addr,
+                    key: key.map(|s| s.to_string()),
+                    profile_image: photo.map(|s| s.to_string()),
+                    timestamp: datetime
+                        .context("No timestamp in vcard")
+                        .and_then(parse_datetime),
+                });
                 break;
             }
         }
-
-        let (authname, addr) =
-            sanitize_name_and_addr(display_name.unwrap_or(""), addr.unwrap_or(""));
-
-        contacts.push(VcardContact {
-            authname,
-            addr,
-            key: key.map(|s| s.to_string()),
-            profile_image: photo.map(|s| s.to_string()),
-            timestamp: datetime
-                .context("No timestamp in vcard")
-                .and_then(parse_datetime),
-        });
     }
 
     contacts
@@ -539,6 +539,30 @@ END:VCARD",
         assert_eq!(contacts.len(), 1);
     }
 
+    #[test]
+    fn test_vcard_with_trailing_newline() {
+        let contacts = parse_vcard(
+            "BEGIN:VCARD\r
+VERSION:4.0\r
+FN:Alice Wonderland\r
+N:Wonderland;Alice;;;Ms.\r
+GENDER:W\r
+EMAIL;TYPE=work:alice@example.com\r
+KEY;TYPE=PGP;ENCODING=b:[base64-data]\r
+REV:20240418T184242Z\r
+END:VCARD\r
+\r",
+        );
+
+        assert_eq!(contacts[0].addr, "alice@example.com".to_string());
+        assert_eq!(contacts[0].authname, "Alice Wonderland".to_string());
+        assert_eq!(contacts[0].key, Some("[base64-data]".to_string()));
+        assert_eq!(contacts[0].profile_image, None);
+        assert_eq!(*contacts[0].timestamp.as_ref().unwrap(), 1713465762);
+
+        assert_eq!(contacts.len(), 1);
+    }
+
     #[test]
     fn test_make_and_parse_vcard() {
         let contacts = [
@@ -558,20 +582,20 @@ END:VCARD",
             },
         ];
         let items = [
-            "BEGIN:VCARD\n\
-             VERSION:4.0\n\
-             EMAIL:alice@example.org\n\
-             FN:Alice Wonderland\n\
-             KEY:data:application/pgp-keys;base64,[base64-data]\n\
-             PHOTO:data:image/jpeg;base64,image in Base64\n\
-             REV:20240418T184242Z\n\
-             END:VCARD\n",
-            "BEGIN:VCARD\n\
-             VERSION:4.0\n\
-             EMAIL:bob@example.com\n\
-             FN:bob@example.com\n\
-             REV:19700101T000000Z\n\
-             END:VCARD\n",
+            "BEGIN:VCARD\r\n\
+             VERSION:4.0\r\n\
+             EMAIL:alice@example.org\r\n\
+             FN:Alice Wonderland\r\n\
+             KEY:data:application/pgp-keys;base64,[base64-data]\r\n\
+             PHOTO:data:image/jpeg;base64,image in Base64\r\n\
+             REV:20240418T184242Z\r\n\
+             END:VCARD\r\n",
+            "BEGIN:VCARD\r\n\
+             VERSION:4.0\r\n\
+             EMAIL:bob@example.com\r\n\
+             FN:bob@example.com\r\n\
+             REV:19700101T000000Z\r\n\
+             END:VCARD\r\n",
         ];
         let mut expected = "".to_string();
         for len in 0..=contacts.len() {

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.149.0"
+version = "1.155.4"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"
@@ -15,7 +15,7 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { workspace = true, default-features = false }
-deltachat-jsonrpc = { workspace = true, optional = true }
+deltachat-jsonrpc = { workspace = true }
 libc = { workspace = true }
 human-panic = { version = "2", default-features = false }
 num-traits = { workspace = true }
@@ -30,5 +30,4 @@ yerpc = { workspace = true, features = ["anyhow_expose"] }
 [features]
 default = ["vendored"]
 vendored = ["deltachat/vendored", "deltachat-jsonrpc/vendored"]
-jsonrpc = ["dep:deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -418,7 +418,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
  *                    1=send and request read receipts
- *                    default=send and request read receipts, only send but not reuqest if `bot` is set
+ *                    default=send and request read receipts, only send but not request if `bot` is set
  * - `bcc_self`     = 0=do not send a copy of outgoing messages to self,
  *                    1=send a copy of outgoing messages to self (default).
  *                    Sending messages to self is needed for a proper multi-account setup,
@@ -722,12 +722,6 @@ char*           dc_get_connectivity_html     (dc_context_t* context);
 int              dc_get_push_state           (dc_context_t* context);
 
 
-/**
- * Only used by the python tests.
- */
-int             dc_all_work_done             (dc_context_t* context);
-
-
 // connect
 
 /**
@@ -969,54 +963,6 @@ uint32_t        dc_create_chat_by_contact_id (dc_context_t* context, uint32_t co
 uint32_t        dc_get_chat_id_by_contact_id (dc_context_t* context, uint32_t contact_id);
 
 
-/**
- * Prepare a message for sending.
- *
- * Call this function if the file to be sent is still in creation.
- * Once you're done with creating the file, call dc_send_msg() as usual
- * and the message will really be sent.
- *
- * This is useful as the user can already send the next messages while
- * e.g. the recoding of a video is not yet finished. Or the user can even forward
- * the message with the file being still in creation to other groups.
- *
- * Files being sent with the increation-method must be placed in the
- * blob directory, see dc_get_blobdir().
- * If the increation-method is not used - which is probably the normal case -
- * dc_send_msg() copies the file to the blob directory if it is not yet there.
- * To distinguish the two cases, msg->state must be set properly. The easiest
- * way to ensure this is to re-use the same object for both calls.
- *
- * Example:
- * ~~~
- * char* blobdir = dc_get_blobdir(context);
- * char* file_to_send = mprintf("%s/%s", blobdir, "send.mp4")
- *
- * dc_msg_t* msg = dc_msg_new(context, DC_MSG_VIDEO);
- * dc_msg_set_file(msg, file_to_send, NULL);
- * dc_prepare_msg(context, chat_id, msg);
- *
- * // ... create the file ...
- *
- * dc_send_msg(context, chat_id, msg);
- *
- * dc_msg_unref(msg);
- * free(file_to_send);
- * dc_str_unref(file_to_send);
- * ~~~
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param chat_id The chat ID to send the message to.
- * @param msg The message object to send to the chat defined by the chat ID.
- *     On success, msg_id and state of the object are set up,
- *     The function does not take ownership of the object,
- *     so you have to free it using dc_msg_unref() as usual.
- * @return The ID of the message that is being prepared.
- */
-uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
-
-
 /**
  * Send a message defined by a dc_msg_t object to a chat.
  *
@@ -1041,13 +987,11 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * If that fails, is not possible, or the image is already small enough, the image is sent as original.
  * If you want images to be always sent as the original file, use the #DC_MSG_FILE type.
  *
- * Videos and other file types are currently not recoded by the library,
- * with dc_prepare_msg(), however, you can do that from the UI.
+ * Videos and other file types are currently not recoded by the library.
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to send the message to.
- *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
  *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
@@ -1064,7 +1008,6 @@ uint32_t        dc_send_msg                  (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to send the message to.
- *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
  *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
@@ -1154,9 +1097,14 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id The ID of the message with the webxdc instance.
- * @param json program-readable data, the actual payload
- * @param descr The user-visible description of JSON data,
- *     in case of a chess game, e.g. the move.
+ * @param json program-readable data, this is created in JS land as:
+ *     - `payload`: any JS object or primitive.
+ *     - `info`: optional informational message. Will be shown in chat and may be added as system notification.
+ *       note that also users that are not notified explicitly get the `info` or `summary` update shown in the chat.
+ *     - `document`: optional document name. shown eg. in title bar.
+ *     - `summary`: optional summary. shown beside app icon.
+ *     - `notify`: optional array of other users `selfAddr` to be notified e.g. by a sound about `info` or `summary`.
+ * @param descr Deprecated, set to NULL
  * @return 1=success, 0=error
  */
 int dc_send_webxdc_status_update (dc_context_t* context, uint32_t msg_id, const char* json, const char* descr);
@@ -2026,6 +1974,36 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 void            dc_forward_msgs              (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt, uint32_t chat_id);
 
 
+/**
+ * Save a copy of messages in "Saved Messages".
+ *
+ * In contrast to forwarding messages,
+ * information as author, date and origin are preserved.
+ * The action completes locally, so "Saved Messages" do not show sending errors in case one is offline.
+ * Still, a sync message is emitted, so that other devices will save the same message,
+ * as long as not deleted before.
+ *
+ * To check if a message was saved, use dc_msg_get_saved_msg_id(),
+ * UI may show an indicator and offer an "Unsave" instead of a "Save" button then.
+ *
+ * The other way round, from inside the "Saved Messages" chat,
+ * UI may show the indicator and "Unsave" button checking dc_msg_get_original_msg_id()
+ * and offer a button to go the original message.
+ *
+ * "Unsave" is done by deleting the saved message.
+ * Webxdc updates are not copied on purpose.
+ *
+ * For performance reasons, esp. when saving lots of messages,
+ * UI should call this function from a background thread.
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param msg_ids An array of uint32_t containing all message IDs that should be saved.
+ * @param msg_cnt The number of messages IDs in the msg_ids array.
+ */
+void            dc_save_msgs                 (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
+
+
 /**
  * Resend messages and make information available for newly added chat members.
  * Resending sends out the original message, however, recipients and webxdc-status may differ.
@@ -2541,6 +2519,8 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  * - DC_QR_PROXY with dc_lot_t::text1=address:
  *   ask the user if they want to use the given proxy.
  *   if so, call dc_set_config_from_qr() and restart I/O.
+ *   On success, dc_get_config(context, "proxy_url")
+ *   will contain the new proxy in normalized form as the first element.
  *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
  *   e-mail address scanned, optionally, a draft message could be set in
@@ -2590,13 +2570,15 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
 
 /**
  * Get QR code text that will offer an Setup-Contact or Verified-Group invitation.
- * The QR code is compatible to the OPENPGP4FPR format
- * so that a basic fingerprint comparison also works e.g. with OpenKeychain.
  *
  * The scanning device will pass the scanned content to dc_check_qr() then;
  * if dc_check_qr() returns DC_QR_ASK_VERIFYCONTACT or DC_QR_ASK_VERIFYGROUP
  * an out-of-band-verification can be joined using dc_join_securejoin()
  *
+ * The returned text will also work as a normal https:-link,
+ * so that the QR code is useful also without Delta Chat being installed
+ * or can be passed to contacts through other channels.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id If set to a group-chat-id,
@@ -3982,7 +3964,7 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *
  * Outgoing message states:
  * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
- *   the message enters this state before @ref DC_STATE_OUT_PENDING.
+ *   the message enters this state before @ref DC_STATE_OUT_PENDING. Deprecated.
  * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
  * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
@@ -4192,9 +4174,13 @@ char*             dc_msg_get_webxdc_blob      (const dc_msg_t* msg, const char*
  *   defaults to an empty string.
  *   Implementations may offer an menu or a button to open this URL.
  * - internet_access:
- *   true if the Webxdc should get full internet access, including Webrtc.
- *   currently, this is only true for encrypted Webxdc's in the self chat
- *   that have requested internet access in the manifest.
+ *   true if the Webxdc should get internet access;
+ *   this is the case i.e. for experimental maps integration.
+ * - self_addr: address to be used for `window.webxdc.selfAddr` in JS land.
+ * - send_update_interval: Milliseconds to wait before calling `sendUpdate()` again since the last call.
+ *   Should be exposed to `webxdc.sendUpdateInterval` in JS land.
+ * - send_update_max_size: Maximum number of bytes accepted for a serialized update object.
+ *   Should be exposed to `webxdc.sendUpdateMaxSize` in JS land.
  *
  * @memberof dc_msg_t
  * @param msg The webxdc instance.
@@ -4480,6 +4466,7 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
  *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
  *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
+ * - DC_INFO_WEBXDC_INFO_MESSAGE (32) - Info-message created by webxdc app sending `update.info`
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4509,19 +4496,23 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
+
 /**
- * Check if a message is still in creation. A message is in creation between
- * the calls to dc_prepare_msg() and dc_send_msg().
+ * Get link attached to an webxdc info message.
+ * The info message needs to be of type DC_INFO_WEBXDC_INFO_MESSAGE.
  *
- * Typically, this is used for videos that are recoded by the UI before
- * they can be sent.
+ * Typically, this is used to set `document.location.href` in JS land.
+ *
+ * Webxdc apps can define the link by setting `update.href` when sending and update,
+ * see dc_send_webxdc_status_update().
  *
  * @memberof dc_msg_t
- * @param msg The message object.
- * @return 1=message is still in creation (dc_send_msg() was not called yet),
- *     0=message no longer in creation.
+ * @param msg The info message object.
+ *     Not: the webxdc instance.
+ * @return The link to be set to `document.location.href` in JS land.
+ *     Returns NULL if there is no link attached to the info message and on errors.
  */
-int             dc_msg_is_increation          (const dc_msg_t* msg);
+char*           dc_msg_get_webxdc_href        (const dc_msg_t* msg);
 
 
 /**
@@ -4674,7 +4665,7 @@ int dc_msg_has_html (dc_msg_t* msg);
   *                                     If the download fails or succeeds,
   *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
   *
-  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any further download action.
   *                                     It was fully downloaded, but we failed to decrypt it.
   * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
@@ -4761,10 +4752,36 @@ void            dc_msg_set_override_sender_name(dc_msg_t* msg, const char* name)
  * @param file If the message object is used in dc_send_msg() later,
  *     this must be the full path of the image file to send.
  * @param filemime The MIME type of the file. NULL if you don't know or don't care.
+ * @deprecated 2025-01-21 Use dc_msg_set_file_and_deduplicate instead
  */
 void            dc_msg_set_file               (dc_msg_t* msg, const char* file, const char* filemime);
 
 
+/**
+ * Sets the file associated with a message.
+ *
+ * If `name` is non-null, it is used as the file name
+ * and the actual current name of the file is ignored.
+ *
+ * If the source file is already in the blobdir, it will be renamed,
+ * otherwise it will be copied to the blobdir first.
+ *
+ * In order to deduplicate files that contain the same data,
+ * the file will be named `<hash>.<extension>`, e.g. `ce940175885d7b78f7b7e9f1396611f.jpg`.
+ *
+ * NOTE:
+ * - This function will rename the file. To get the new file path, call `get_file()`.
+ * - The file must not be modified after this function was called.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object. Must not be NULL.
+ * @param file The path of the file to attach. Must not be NULL.
+ * @param name The original filename of the attachment. If NULL, the current name of `file` will be used instead.
+ * @param filemime The MIME type of the file. NULL if you don't know or don't care.
+ */
+void            dc_msg_set_file_and_deduplicate(dc_msg_t* msg, const char* file, const char* name, const char* filemime);
+
+
 /**
  * Set the dimensions associated with message object.
  * Typically this is the width and the height of an image or video associated using dc_msg_set_file().
@@ -4907,6 +4924,35 @@ dc_msg_t*       dc_msg_get_quoted_msg         (const dc_msg_t* msg);
 dc_msg_t*       dc_msg_get_parent             (const dc_msg_t* msg);
 
 
+/**
+ * Get original message ID for a saved message from the "Saved Messages" chat.
+ *
+ * Can be used by UI to show a button to go the original message
+ * and an option to "Unsave" the message.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object. Usually, this refers to a a message inside "Saved Messages".
+ * @return The message ID of the original message.
+ *     0 if the given message object is not a "Saved Message"
+ *     or if the original message does no longer exist.
+ */
+uint32_t        dc_msg_get_original_msg_id    (const dc_msg_t* msg);
+
+
+/**
+ * Check if a message was saved and return its ID inside "Saved Messages".
+ *
+ * Deleting the returned message will un-save the message.
+ * The state "is saved" can be used to show some icon to indicate that a message was saved.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object. Usually, this refers to a a message outside "Saved Messages".
+ * @return The message ID inside "Saved Messages", if any.
+ *     0 if the given message object is not saved.
+ */
+uint32_t        dc_msg_get_saved_msg_id     (const dc_msg_t* msg);
+
+
 /**
  * Force the message to be sent in plain text.
  *
@@ -5432,6 +5478,8 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Message containing a sticker, similar to image.
+ * NB: When sending, the message viewtype may be changed to `Image` by some heuristics like checking
+ * for transparent pixels.
  * If possible, the UI should display the image without borders in a transparent way.
  * A click on a sticker will offer to install the sticker set in some future.
  */
@@ -5536,6 +5584,8 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Outgoing message being prepared. See dc_msg_get_state() for details.
+ *
+ * @deprecated 2024-12-07
  */
 #define         DC_STATE_OUT_PREPARING       18
 
@@ -5709,8 +5759,14 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_CERTCK_STRICT 1
 
 /**
- * Accept invalid certificates, including self-signed ones
- * or having incorrect hostname.
+ * Accept certificates that are expired, self-signed
+ * or not valid for the server hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID 2
+
+/**
+ * For API compatibility only: Treat this as DC_CERTCK_ACCEPT_INVALID on reading.
+ * Must not be written.
  */
 #define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
 
@@ -5750,6 +5806,23 @@ void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);
  * returns immediately and once the result is ready it can be retrieved via dc_jsonrpc_next_response()
  * the jsonrpc specification defines an invocation id that can then be used to match request and response.
  *
+ * An overview of JSON-RPC calls is available at
+ * <https://js.jsonrpc.delta.chat/classes/RawClient.html>.
+ * Note that the page describes only the rough methods.
+ * Calling convention, casing etc. does vary, this is a known flaw,
+ * and at some point we will get to improve that :)
+ *
+ * Also, note that most calls are more high-level than this CFFI, require more database calls and are slower.
+ * They're more suitable for an environment that is totally async and/or cannot use CFFI, which might not be true for native apps.
+ *
+ * Notable exceptions that exist only as JSON-RPC and probably never get a CFFI counterpart:
+ * - getMessageReactions(), sendReaction()
+ * - getHttpResponse()
+ * - draftSelfReport()
+ * - getAccountFileSize()
+ * - importVcard(), parseVcard(), makeVcard()
+ * - sendWebxdcRealtimeData, sendWebxdcRealtimeAdvertisement(), leaveWebxdcRealtime()
+ *
  * @memberof dc_jsonrpc_instance_t
  * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
  * @param request JSON-RPC request as string
@@ -5770,6 +5843,8 @@ char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);
 /**
  * Make a JSON-RPC call and return a response.
  *
+ * See dc_jsonrpc_request() for an overview of possible calls and for more information.
+ *
  * @memberof dc_jsonrpc_instance_t
  * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
  * @param input JSON-RPC request.
@@ -5865,15 +5940,26 @@ int dc_event_get_data2_int(dc_event_t* event);
 
 /**
  * Get data associated with an event object.
- * The meaning of the data depends on the event ID
- * returned as @ref DC_EVENT constants by dc_event_get_id().
- * See also dc_event_get_data1_int() and dc_event_get_data2_int().
+ * The meaning of the data depends on the event ID returned as @ref DC_EVENT constants.
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return "data2" as a string or NULL.
- *     the meaning depends on the event type associated with this event.
- *     Once you're done with the string, you have to unref it using dc_unref_str().
+ * @return "data1" string or NULL.
+ *     The meaning depends on the event type associated with this event.
+ *     Must be freed using dc_str_unref().
+ */
+char* dc_event_get_data1_str(dc_event_t* event);
+
+
+/**
+ * Get data associated with an event object.
+ * The meaning of the data depends on the event ID returned as @ref DC_EVENT constants.
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" string or NULL.
+ *     The meaning depends on the event type associated with this event.
+ *     Must be freed using dc_str_unref().
  */
 char* dc_event_get_data2_str(dc_event_t* event);
 
@@ -6071,12 +6157,35 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_INCOMING_REACTION        2002
 
 
+
+/**
+ * A webxdc wants an info message or a changed summary to be notified.
+ *
+ * @param data1 (int) contact_id ID _and_ (char*) href.
+ *      - dc_event_get_data1_int() returns contact_id of the sending contact.
+ *      - dc_event_get_data1_str() returns the href as set to `update.href`.
+ * @param data2 (int) msg_id _and_ (char*) text_to_notify.
+ *      - dc_event_get_data2_int() returns the msg_id,
+ *        referring to the webxdc-info-message, if there is any.
+ *        Sometimes no webxdc-info-message is added to the chat
+ *        and yet a notification is sent; in this case the msg_id
+ *        of the webxdc instance is returned.
+ *      - dc_event_get_data2_str() returns text_to_notify,
+ *        the text that shall be shown in the notification.
+ *        string must be passed to dc_str_unref() afterwards.
+ */
+#define DC_EVENT_INCOMING_WEBXDC_NOTIFY   2003
+
+
 /**
  * There is a fresh message. Typically, the user will show an notification
  * when receiving this message.
  *
  * There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
  *
+ * If the message is a webxdc info message,
+ * dc_msg_get_parent() returns the webxdc instance the notification belongs to.
+ *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
  */
@@ -6360,6 +6469,25 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_CHATLIST_ITEM_CHANGED         2301
 
+/**
+ * Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+ *
+ * This event is only emitted by the account manager.
+ */
+
+#define DC_EVENT_ACCOUNTS_CHANGED              2302
+
+/**
+ * Inform that an account property that might be shown in the account list changed, namely:
+ * - is_configured (see dc_is_configured())
+ * - displayname
+ * - selfavatar
+ * - private_tag
+ * 
+ * This event is emitted from the account whose property changed.
+ */
+
+#define DC_EVENT_ACCOUNTS_ITEM_CHANGED         2303
 
 /**
  * Inform that some events have been skipped due to event channel overflow.
@@ -6807,7 +6935,7 @@ void dc_event_unref(dc_event_t* event);
 
 /// "Failed to send message to %1$s."
 ///
-/// Used in status messages.
+/// Unused. Was used in group chat status messages.
 /// - %1$s will be replaced by the name of the contact the message cannot be sent to
 #define DC_STR_FAILED_SENDING_TO          74
 

deltachat-ffi/src/lib.rs

@@ -35,6 +35,8 @@ use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
+use deltachat_jsonrpc::api::CommandApi;
+use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession};
 use num_traits::{FromPrimitive, ToPrimitive};
 use once_cell::sync::Lazy;
 use rand::Rng;
@@ -413,16 +415,6 @@ pub unsafe extern "C" fn dc_get_push_state(context: *const dc_context_t) -> libc
     block_on(ctx.push_state()) as libc::c_int
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_all_work_done(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_all_work_done()");
-        return 0;
-    }
-    let ctx = &*context;
-    block_on(async move { ctx.all_work_done().await as libc::c_int })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_oauth2_url(
     context: *mut dc_context_t,
@@ -542,6 +534,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::MsgsChanged { .. } => 2000,
         EventType::ReactionsChanged { .. } => 2001,
         EventType::IncomingReaction { .. } => 2002,
+        EventType::IncomingWebxdcNotify { .. } => 2003,
         EventType::IncomingMsg { .. } => 2005,
         EventType::IncomingMsgBunch { .. } => 2006,
         EventType::MsgsNoticed { .. } => 2008,
@@ -568,6 +561,8 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::AccountsBackgroundFetchDone => 2200,
         EventType::ChatlistChanged => 2300,
         EventType::ChatlistItemChanged { .. } => 2301,
+        EventType::AccountsChanged => 2302,
+        EventType::AccountsItemChanged => 2303,
         EventType::EventChannelOverflow { .. } => 2400,
         #[allow(unreachable_patterns)]
         #[cfg(test)]
@@ -600,9 +595,12 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ErrorSelfNotInGroup(_)
-        | EventType::AccountsBackgroundFetchDone => 0,
-        EventType::ChatlistChanged => 0,
-        EventType::IncomingReaction { contact_id, .. } => contact_id.to_u32() as libc::c_int,
+        | EventType::AccountsBackgroundFetchDone
+        | EventType::ChatlistChanged
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged => 0,
+        EventType::IncomingReaction { contact_id, .. }
+        | EventType::IncomingWebxdcNotify { contact_id, .. } => contact_id.to_u32() as libc::c_int,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -674,6 +672,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatlistChanged
         | EventType::ChatlistItemChanged { .. }
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged
         | EventType::ConfigSynced { .. }
         | EventType::ChatModified(_)
         | EventType::WebxdcRealtimeAdvertisementReceived { .. }
@@ -681,6 +681,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
         | EventType::IncomingReaction { msg_id, .. }
+        | EventType::IncomingWebxdcNotify { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
         | EventType::MsgFailed { msg_id, .. }
@@ -700,6 +701,27 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_event_get_data1_str(event: *mut dc_event_t) -> *mut libc::c_char {
+    if event.is_null() {
+        eprintln!("ignoring careless call to dc_event_get_data1_str()");
+        return ptr::null_mut();
+    }
+
+    let event = &(*event).typ;
+
+    match event {
+        EventType::IncomingWebxdcNotify { href, .. } => {
+            if let Some(href) = href {
+                href.to_c_string().unwrap_or_default().into_raw()
+            } else {
+                ptr::null_mut()
+            }
+        }
+        _ => ptr::null_mut(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut libc::c_char {
     if event.is_null() {
@@ -748,6 +770,8 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::IncomingMsgBunch { .. }
         | EventType::ChatlistItemChanged { .. }
         | EventType::ChatlistChanged
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged
         | EventType::WebxdcRealtimeAdvertisementReceived { .. }
         | EventType::EventChannelOverflow { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
@@ -775,6 +799,9 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             .to_c_string()
             .unwrap_or_default()
             .into_raw(),
+        EventType::IncomingWebxdcNotify { text, .. } => {
+            text.to_c_string().unwrap_or_default().into_raw()
+        }
         #[allow(unreachable_patterns)]
         #[cfg(test)]
         _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
@@ -951,27 +978,6 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_prepare_msg(
-    context: *mut dc_context_t,
-    chat_id: u32,
-    msg: *mut dc_msg_t,
-) -> u32 {
-    if context.is_null() || chat_id == 0 || msg.is_null() {
-        eprintln!("ignoring careless call to dc_prepare_msg()");
-        return 0;
-    }
-    let ctx = &mut *context;
-    let ffi_msg: &mut MessageWrapper = &mut *msg;
-
-    block_on(async move {
-        chat::prepare_msg(ctx, ChatId::new(chat_id), &mut ffi_msg.message)
-            .await
-            .unwrap_or_log_default(ctx, "Failed to prepare message")
-    })
-    .to_u32()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_send_msg(
     context: *mut dc_context_t,
@@ -1059,7 +1065,7 @@ pub unsafe extern "C" fn dc_send_webxdc_status_update(
     context: *mut dc_context_t,
     msg_id: u32,
     json: *const libc::c_char,
-    descr: *const libc::c_char,
+    _descr: *const libc::c_char,
 ) -> libc::c_int {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_send_webxdc_status_update()");
@@ -1067,14 +1073,10 @@ pub unsafe extern "C" fn dc_send_webxdc_status_update(
     }
     let ctx = &*context;
 
-    block_on(ctx.send_webxdc_status_update(
-        MsgId::new(msg_id),
-        &to_string_lossy(json),
-        &to_string_lossy(descr),
-    ))
-    .context("Failed to send webxdc update")
-    .log_err(ctx)
-    .is_ok() as libc::c_int
+    block_on(ctx.send_webxdc_status_update(MsgId::new(msg_id), &to_string_lossy(json)))
+        .context("Failed to send webxdc update")
+        .log_err(ctx)
+        .is_ok() as libc::c_int
 }
 
 #[no_mangle]
@@ -1977,6 +1979,26 @@ pub unsafe extern "C" fn dc_forward_msgs(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_save_msgs(
+    context: *mut dc_context_t,
+    msg_ids: *const u32,
+    msg_cnt: libc::c_int,
+) {
+    if context.is_null() || msg_ids.is_null() || msg_cnt <= 0 {
+        eprintln!("ignoring careless call to dc_save_msgs()");
+        return;
+    }
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
+    let ctx = &*context;
+
+    block_on(async move {
+        chat::save_msgs(ctx, &msg_ids[..])
+            .await
+            .unwrap_or_log_default(ctx, "Failed to save message")
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_resend_msgs(
     context: *mut dc_context_t,
@@ -3682,13 +3704,14 @@ pub unsafe extern "C" fn dc_msg_get_info_type(msg: *mut dc_msg_t) -> libc::c_int
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_msg_is_increation(msg: *mut dc_msg_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_msg_get_webxdc_href(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_is_increation()");
-        return 0;
+        eprintln!("ignoring careless call to dc_msg_get_webxdc_href()");
+        return "".strdup();
     }
+
     let ffi_msg = &*msg;
-    ffi_msg.message.is_increation().into()
+    ffi_msg.message.get_webxdc_href().strdup()
 }
 
 #[no_mangle]
@@ -3812,6 +3835,33 @@ pub unsafe extern "C" fn dc_msg_set_file(
     )
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_set_file_and_deduplicate(
+    msg: *mut dc_msg_t,
+    file: *const libc::c_char,
+    name: *const libc::c_char,
+    filemime: *const libc::c_char,
+) {
+    if msg.is_null() || file.is_null() {
+        eprintln!("ignoring careless call to dc_msg_set_file_and_deduplicate()");
+        return;
+    }
+    let ffi_msg = &mut *msg;
+    let ctx = &*ffi_msg.context;
+
+    ffi_msg
+        .message
+        .set_file_and_deduplicate(
+            ctx,
+            as_path(file),
+            to_opt_string_lossy(name).as_deref(),
+            to_opt_string_lossy(filemime).as_deref(),
+        )
+        .context("Failed to set file")
+        .log_err(&*ffi_msg.context)
+        .ok();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_set_dimension(
     msg: *mut dc_msg_t,
@@ -3977,6 +4027,48 @@ pub unsafe extern "C" fn dc_msg_get_parent(msg: *const dc_msg_t) -> *mut dc_msg_
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_original_msg_id(msg: *const dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_original_msg_id()");
+        return 0;
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    block_on(async move {
+        ffi_msg
+            .message
+            .get_original_msg_id(context)
+            .await
+            .context("failed to get original message")
+            .log_err(context)
+            .unwrap_or_default()
+            .map(|id| id.to_u32())
+            .unwrap_or(0)
+    })
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_saved_msg_id(msg: *const dc_msg_t) -> u32 {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_saved_msg_id()");
+        return 0;
+    }
+    let ffi_msg: &MessageWrapper = &*msg;
+    let context = &*ffi_msg.context;
+    block_on(async move {
+        ffi_msg
+            .message
+            .get_saved_msg_id(context)
+            .await
+            .context("failed to get original message")
+            .log_err(context)
+            .unwrap_or_default()
+            .map(|id| id.to_u32())
+            .unwrap_or(0)
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_force_plaintext(msg: *mut dc_msg_t) {
     if msg.is_null() {
@@ -4929,105 +5021,97 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     Box::into_raw(Box::new(emitter))
 }
 
-#[cfg(feature = "jsonrpc")]
-mod jsonrpc {
-    use deltachat_jsonrpc::api::CommandApi;
-    use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession};
+pub struct dc_jsonrpc_instance_t {
+    receiver: OutReceiver,
+    handle: RpcSession<CommandApi>,
+}
 
-    use super::*;
-
-    pub struct dc_jsonrpc_instance_t {
-        receiver: OutReceiver,
-        handle: RpcSession<CommandApi>,
+#[no_mangle]
+pub unsafe extern "C" fn dc_jsonrpc_init(
+    account_manager: *mut dc_accounts_t,
+) -> *mut dc_jsonrpc_instance_t {
+    if account_manager.is_null() {
+        eprintln!("ignoring careless call to dc_jsonrpc_init()");
+        return ptr::null_mut();
     }
 
-    #[no_mangle]
-    pub unsafe extern "C" fn dc_jsonrpc_init(
-        account_manager: *mut dc_accounts_t,
-    ) -> *mut dc_jsonrpc_instance_t {
-        if account_manager.is_null() {
-            eprintln!("ignoring careless call to dc_jsonrpc_init()");
-            return ptr::null_mut();
-        }
+    let account_manager = &*account_manager;
+    let cmd_api = block_on(deltachat_jsonrpc::api::CommandApi::from_arc(
+        account_manager.inner.clone(),
+    ));
 
-        let account_manager = &*account_manager;
-        let cmd_api = block_on(deltachat_jsonrpc::api::CommandApi::from_arc(
-            account_manager.inner.clone(),
-        ));
+    let (request_handle, receiver) = RpcClient::new();
+    let handle = RpcSession::new(request_handle, cmd_api);
 
-        let (request_handle, receiver) = RpcClient::new();
-        let handle = RpcSession::new(request_handle, cmd_api);
+    let instance = dc_jsonrpc_instance_t { receiver, handle };
 
-        let instance = dc_jsonrpc_instance_t { receiver, handle };
+    Box::into_raw(Box::new(instance))
+}
 
-        Box::into_raw(Box::new(instance))
+#[no_mangle]
+pub unsafe extern "C" fn dc_jsonrpc_unref(jsonrpc_instance: *mut dc_jsonrpc_instance_t) {
+    if jsonrpc_instance.is_null() {
+        eprintln!("ignoring careless call to dc_jsonrpc_unref()");
+        return;
+    }
+    drop(Box::from_raw(jsonrpc_instance));
+}
+
+fn spawn_handle_jsonrpc_request(handle: RpcSession<CommandApi>, request: String) {
+    spawn(async move {
+        handle.handle_incoming(&request).await;
+    });
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_jsonrpc_request(
+    jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+    request: *const libc::c_char,
+) {
+    if jsonrpc_instance.is_null() || request.is_null() {
+        eprintln!("ignoring careless call to dc_jsonrpc_request()");
+        return;
     }
 
-    #[no_mangle]
-    pub unsafe extern "C" fn dc_jsonrpc_unref(jsonrpc_instance: *mut dc_jsonrpc_instance_t) {
-        if jsonrpc_instance.is_null() {
-            eprintln!("ignoring careless call to dc_jsonrpc_unref()");
-            return;
-        }
-        drop(Box::from_raw(jsonrpc_instance));
+    let handle = &(*jsonrpc_instance).handle;
+    let request = to_string_lossy(request);
+    spawn_handle_jsonrpc_request(handle.clone(), request);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_jsonrpc_next_response(
+    jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+) -> *mut libc::c_char {
+    if jsonrpc_instance.is_null() {
+        eprintln!("ignoring careless call to dc_jsonrpc_next_response()");
+        return ptr::null_mut();
     }
+    let api = &*jsonrpc_instance;
+    block_on(api.receiver.recv())
+        .map(|result| serde_json::to_string(&result).unwrap_or_default().strdup())
+        .unwrap_or(ptr::null_mut())
+}
 
-    fn spawn_handle_jsonrpc_request(handle: RpcSession<CommandApi>, request: String) {
-        spawn(async move {
-            handle.handle_incoming(&request).await;
-        });
+#[no_mangle]
+pub unsafe extern "C" fn dc_jsonrpc_blocking_call(
+    jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+    input: *const libc::c_char,
+) -> *mut libc::c_char {
+    if jsonrpc_instance.is_null() {
+        eprintln!("ignoring careless call to dc_jsonrpc_blocking_call()");
+        return ptr::null_mut();
     }
-
-    #[no_mangle]
-    pub unsafe extern "C" fn dc_jsonrpc_request(
-        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
-        request: *const libc::c_char,
-    ) {
-        if jsonrpc_instance.is_null() || request.is_null() {
-            eprintln!("ignoring careless call to dc_jsonrpc_request()");
-            return;
-        }
-
-        let handle = &(*jsonrpc_instance).handle;
-        let request = to_string_lossy(request);
-        spawn_handle_jsonrpc_request(handle.clone(), request);
-    }
-
-    #[no_mangle]
-    pub unsafe extern "C" fn dc_jsonrpc_next_response(
-        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
-    ) -> *mut libc::c_char {
-        if jsonrpc_instance.is_null() {
-            eprintln!("ignoring careless call to dc_jsonrpc_next_response()");
-            return ptr::null_mut();
-        }
-        let api = &*jsonrpc_instance;
-        block_on(api.receiver.recv())
-            .map(|result| serde_json::to_string(&result).unwrap_or_default().strdup())
-            .unwrap_or(ptr::null_mut())
-    }
-
-    #[no_mangle]
-    pub unsafe extern "C" fn dc_jsonrpc_blocking_call(
-        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
-        input: *const libc::c_char,
-    ) -> *mut libc::c_char {
-        if jsonrpc_instance.is_null() {
-            eprintln!("ignoring careless call to dc_jsonrpc_blocking_call()");
-            return ptr::null_mut();
-        }
-        let api = &*jsonrpc_instance;
-        let input = to_string_lossy(input);
-        let res = block_on(api.handle.process_incoming(&input));
-        match res {
-            Some(message) => {
-                if let Ok(message) = serde_json::to_string(&message) {
-                    message.strdup()
-                } else {
-                    ptr::null_mut()
-                }
+    let api = &*jsonrpc_instance;
+    let input = to_string_lossy(input);
+    let res = block_on(api.handle.process_incoming(&input));
+    match res {
+        Some(message) => {
+            if let Ok(message) = serde_json::to_string(&message) {
+                message.strdup()
+            } else {
+                ptr::null_mut()
             }
-            None => ptr::null_mut(),
         }
+        None => ptr::null_mut(),
     }
 }

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.149.0"
+version = "1.155.4"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -33,7 +33,7 @@ base64 = { workspace = true }
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.11.5", optional = true }
+env_logger = { version = "0.11.6", optional = true }
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["full", "rt-multi-thread"] }

deltachat-jsonrpc/src/api.rs

@@ -45,7 +45,7 @@ pub mod types;
 
 use num_traits::FromPrimitive;
 use types::account::Account;
-use types::chat::{EncryptionInfo, FullChat};
+use types::chat::FullChat;
 use types::contact::{ContactObject, VcardContact};
 use types::events::Event;
 use types::http::HttpResponse;
@@ -212,14 +212,12 @@ impl CommandApi {
         self.accounts.read().await.get_all()
     }
 
-    /// Select account id for internally selected state.
-    /// TODO: Likely this is deprecated as all methods take an account id now.
+    /// Select account in account manager, this saves the last used account to accounts.toml
     async fn select_account(&self, id: u32) -> Result<()> {
         self.accounts.write().await.select_account(id).await
     }
 
-    /// Get the selected account id of the internal state..
-    /// TODO: Likely this is deprecated as all methods take an account id now.
+    /// Get the selected account from the account manager (on startup it is read from accounts.toml)
     async fn get_selected_account_id(&self) -> Option<u32> {
         self.accounts.read().await.get_selected_account_id()
     }
@@ -708,19 +706,6 @@ impl CommandApi {
         ChatId::new(chat_id).get_encryption_info(&ctx).await
     }
 
-    /// Get encryption info for a chat.
-    async fn get_chat_encryption_info_json(
-        &self,
-        account_id: u32,
-        chat_id: u32,
-    ) -> Result<EncryptionInfo> {
-        let ctx = self.get_context(account_id).await?;
-        Ok(ChatId::new(chat_id)
-            .get_encryption_info_json(&ctx)
-            .await?
-            .into())
-    }
-
     /// Get QR code text that will offer a [SecureJoin](https://securejoin.delta.chat/) invitation.
     ///
     /// If `chat_id` is a group chat ID, SecureJoin QR code for the group is returned.
@@ -849,6 +834,13 @@ impl CommandApi {
         Ok(contacts.iter().map(|id| id.to_u32()).collect::<Vec<u32>>())
     }
 
+    /// Returns contact IDs of the past chat members.
+    async fn get_past_chat_contacts(&self, account_id: u32, chat_id: u32) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts = chat::get_past_chat_contacts(&ctx, ChatId::new(chat_id)).await?;
+        Ok(contacts.iter().map(|id| id.to_u32()).collect::<Vec<u32>>())
+    }
+
     /// Create a new group chat.
     ///
     /// After creation,
@@ -1006,6 +998,12 @@ impl CommandApi {
         marknoticed_chat(&ctx, ChatId::new(chat_id)).await
     }
 
+    /// Returns the message that is immediately followed by the last seen
+    /// message.
+    /// From the point of view of the user this is effectively
+    /// "first unread", but in reality in the database a seen message
+    /// _can_ be followed by a fresh (unseen) message
+    /// if that message has not been individually marked as seen.
     async fn get_first_unread_message_of_chat(
         &self,
         account_id: u32,
@@ -1094,6 +1092,9 @@ impl CommandApi {
         markseen_msgs(&ctx, msg_ids.into_iter().map(MsgId::new).collect()).await
     }
 
+    /// Returns all messages of a particular chat.
+    /// If `add_daymarker` is `true`, it will return them as
+    /// `DC_MSG_ID_DAYMARKER`, e.g. [1234, 1237, 9, 1239].
     async fn get_message_ids(
         &self,
         account_id: u32,
@@ -1148,9 +1149,11 @@ impl CommandApi {
     async fn get_message(&self, account_id: u32, msg_id: u32) -> Result<MessageObject> {
         let ctx = self.get_context(account_id).await?;
         let msg_id = MsgId::new(msg_id);
-        MessageObject::from_msg_id(&ctx, msg_id)
+        let message_object = MessageObject::from_msg_id(&ctx, msg_id)
             .await
-            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))
+            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))?
+            .with_context(|| format!("Message {msg_id} does not exist for account {account_id}"))?;
+        Ok(message_object)
     }
 
     async fn get_message_html(&self, account_id: u32, message_id: u32) -> Result<Option<String>> {
@@ -1174,7 +1177,10 @@ impl CommandApi {
             messages.insert(
                 message_id,
                 match message_result {
-                    Ok(message) => MessageLoadResult::Message(message),
+                    Ok(Some(message)) => MessageLoadResult::Message(message),
+                    Ok(None) => MessageLoadResult::LoadingError {
+                        error: "Message does not exist".to_string(),
+                    },
                     Err(error) => MessageLoadResult::LoadingError {
                         error: format!("{error:#}"),
                     },
@@ -1775,10 +1781,10 @@ impl CommandApi {
         account_id: u32,
         instance_msg_id: u32,
         update_str: String,
-        description: String,
+        _descr: Option<String>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        ctx.send_webxdc_status_update(MsgId::new(instance_msg_id), &update_str, &description)
+        ctx.send_webxdc_status_update(MsgId::new(instance_msg_id), &update_str)
             .await
     }
 
@@ -1837,6 +1843,18 @@ impl CommandApi {
         WebxdcMessageInfo::get_for_message(&ctx, MsgId::new(instance_msg_id)).await
     }
 
+    /// Get href from a WebxdcInfoMessage which might include a hash holding
+    /// information about a specific position or state in a webxdc app (optional)
+    async fn get_webxdc_href(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+    ) -> Result<Option<String>> {
+        let ctx = self.get_context(account_id).await?;
+        let message = Message::load_from_db(&ctx, MsgId::new(instance_msg_id)).await?;
+        Ok(message.get_webxdc_href())
+    }
+
     /// Get blob encoded as base64 from a webxdc message
     ///
     /// path is the path of the file within webxdc archive
@@ -2012,9 +2030,7 @@ impl CommandApi {
     async fn get_draft(&self, account_id: u32, chat_id: u32) -> Result<Option<MessageObject>> {
         let ctx = self.get_context(account_id).await?;
         if let Some(draft) = ChatId::new(chat_id).get_draft(&ctx).await? {
-            Ok(Some(
-                MessageObject::from_msg_id(&ctx, draft.get_id()).await?,
-            ))
+            Ok(MessageObject::from_msg_id(&ctx, draft.get_id()).await?)
         } else {
             Ok(None)
         }
@@ -2183,7 +2199,9 @@ impl CommandApi {
                 .await?;
         }
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message).await?;
-        let message = MessageObject::from_msg_id(&ctx, msg_id).await?;
+        let message = MessageObject::from_msg_id(&ctx, msg_id)
+            .await?
+            .context("Just sent message does not exist")?;
         Ok((msg_id.to_u32(), message))
     }
 

deltachat-jsonrpc/src/api/types/chat.rs

@@ -1,7 +1,7 @@
 use std::time::{Duration, SystemTime};
 
 use anyhow::{bail, Context as _, Result};
-use deltachat::chat::{self, get_chat_contacts, ChatVisibility};
+use deltachat::chat::{self, get_chat_contacts, get_past_chat_contacts, ChatVisibility};
 use deltachat::chat::{Chat, ChatId};
 use deltachat::constants::Chattype;
 use deltachat::contact::{Contact, ContactId};
@@ -9,7 +9,6 @@ use deltachat::context::Context;
 use num_traits::cast::ToPrimitive;
 use serde::{Deserialize, Serialize};
 use typescript_type_def::TypeDef;
-use yerpc::JsonSchema;
 
 use super::color_int_to_hex_string;
 use super::contact::ContactObject;
@@ -40,6 +39,10 @@ pub struct FullChat {
     is_self_talk: bool,
     contacts: Vec<ContactObject>,
     contact_ids: Vec<u32>,
+
+    /// Contact IDs of the past chat members.
+    past_contact_ids: Vec<u32>,
+
     color: String,
     fresh_message_counter: usize,
     // is_group - please check over chat.type in frontend instead
@@ -60,6 +63,7 @@ impl FullChat {
         let chat = Chat::load_from_db(context, rust_chat_id).await?;
 
         let contact_ids = get_chat_contacts(context, rust_chat_id).await?;
+        let past_contact_ids = get_past_chat_contacts(context, rust_chat_id).await?;
 
         let mut contacts = Vec::with_capacity(contact_ids.len());
 
@@ -112,6 +116,7 @@ impl FullChat {
             is_self_talk: chat.is_self_talk(),
             contacts,
             contact_ids: contact_ids.iter().map(|id| id.to_u32()).collect(),
+            past_contact_ids: past_contact_ids.iter().map(|id| id.to_u32()).collect(),
             color,
             fresh_message_counter,
             is_contact_request: chat.is_contact_request(),
@@ -240,23 +245,3 @@ impl JSONRPCChatVisibility {
         }
     }
 }
-
-#[derive(Debug, JsonSchema, TypeDef, Serialize, Deserialize)]
-pub struct EncryptionInfo {
-    /// Addresses with End-to-end encryption preferred.
-    pub mutual: Vec<String>,
-    /// Addresses with End-to-end encryption available.
-    pub no_preference: Vec<String>,
-    /// Addresses with no encryption.
-    pub reset: Vec<String>,
-}
-
-impl From<chat::EncryptionInfo> for EncryptionInfo {
-    fn from(encryption_info: chat::EncryptionInfo) -> Self {
-        EncryptionInfo {
-            mutual: encryption_info.mutual,
-            no_preference: encryption_info.no_preference,
-            reset: encryption_info.reset,
-        }
-    }
-}

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -88,11 +88,17 @@ pub(crate) async fn get_chat_list_item_by_id(
 
     let (last_updated, message_type) = match last_msgid {
         Some(id) => {
-            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
-            (
-                Some(last_message.get_timestamp() * 1000),
-                Some(last_message.get_viewtype().into()),
-            )
+            if let Some(last_message) =
+                deltachat::message::Message::load_from_db_optional(ctx, id).await?
+            {
+                (
+                    Some(last_message.get_timestamp() * 1000),
+                    Some(last_message.get_viewtype().into()),
+                )
+            } else {
+                // Message may be deleted by the time we try to load it.
+                (None, None)
+            }
         }
         None => (None, None),
     };

deltachat-jsonrpc/src/api/types/events.rs

@@ -69,7 +69,7 @@ pub enum EventType {
     /// or for functions that are expected to fail (eg. autocryptContinueKeyTransfer())
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
-    /// in a messasge box then.
+    /// in a message box then.
     Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
@@ -101,11 +101,22 @@ pub enum EventType {
     /// Incoming reaction, should be notified.
     #[serde(rename_all = "camelCase")]
     IncomingReaction {
+        chat_id: u32,
         contact_id: u32,
         msg_id: u32,
         reaction: String,
     },
 
+    /// Incoming webxdc info or summary update, should be notified.
+    #[serde(rename_all = "camelCase")]
+    IncomingWebxdcNotify {
+        chat_id: u32,
+        contact_id: u32,
+        msg_id: u32,
+        text: String,
+        href: Option<String>,
+    },
+
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -277,6 +288,20 @@ pub enum EventType {
     #[serde(rename_all = "camelCase")]
     ChatlistItemChanged { chat_id: Option<u32> },
 
+    /// Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+    ///
+    /// This event is only emitted by the account manager
+    AccountsChanged,
+
+    /// Inform that an account property that might be shown in the account list changed, namely:
+    /// - is_configured (see is_configured())
+    /// - displayname
+    /// - selfavatar
+    /// - private_tag
+    ///
+    /// This event is emitted from the account whose property changed.
+    AccountsItemChanged,
+
     /// Inform than some events have been skipped due to event channel overflow.
     EventChannelOverflow { n: u64 },
 }
@@ -311,14 +336,29 @@ impl From<CoreEventType> for EventType {
                 contact_id: contact_id.to_u32(),
             },
             CoreEventType::IncomingReaction {
+                chat_id,
                 contact_id,
                 msg_id,
                 reaction,
             } => IncomingReaction {
+                chat_id: chat_id.to_u32(),
                 contact_id: contact_id.to_u32(),
                 msg_id: msg_id.to_u32(),
                 reaction: reaction.as_str().to_string(),
             },
+            CoreEventType::IncomingWebxdcNotify {
+                chat_id,
+                contact_id,
+                msg_id,
+                text,
+                href,
+            } => IncomingWebxdcNotify {
+                chat_id: chat_id.to_u32(),
+                contact_id: contact_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+                text,
+                href,
+            },
             CoreEventType::IncomingMsg { chat_id, msg_id } => IncomingMsg {
                 chat_id: chat_id.to_u32(),
                 msg_id: msg_id.to_u32(),
@@ -409,6 +449,8 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ChatlistChanged => ChatlistChanged,
             CoreEventType::EventChannelOverflow { n } => EventChannelOverflow { n },
+            CoreEventType::AccountsChanged => AccountsChanged,
+            CoreEventType::AccountsItemChanged => AccountsItemChanged,
             #[allow(unreachable_patterns)]
             #[cfg(test)]
             _ => unreachable!("This is just to silence a rust_analyzer false-positive"),

deltachat-jsonrpc/src/api/types/message.rs

@@ -85,6 +85,8 @@ pub struct MessageObject {
 
     webxdc_info: Option<WebxdcMessageInfo>,
 
+    webxdc_href: Option<String>,
+
     download_state: DownloadState,
 
     reactions: Option<JSONRPCReactions>,
@@ -102,6 +104,9 @@ enum MessageQuote {
     WithMessage {
         text: String,
         message_id: u32,
+        /// The quoted message does not always belong
+        /// to the same chat, e.g. when "Reply Privately" is used.
+        chat_id: u32,
         author_display_name: String,
         author_display_color: String,
         override_sender_name: Option<String>,
@@ -112,8 +117,10 @@ enum MessageQuote {
 }
 
 impl MessageObject {
-    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
-        let message = Message::load_from_db(context, msg_id).await?;
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Option<Self>> {
+        let Some(message) = Message::load_from_db_optional(context, msg_id).await? else {
+            return Ok(None);
+        };
 
         let sender_contact = Contact::get_by_id(context, message.get_from_id())
             .await
@@ -143,6 +150,7 @@ impl MessageObject {
                     Some(MessageQuote::WithMessage {
                         text: quoted_text,
                         message_id: quote.get_id().to_u32(),
+                        chat_id: quote.get_chat_id().to_u32(),
                         author_display_name: quote_author.get_display_name().to_owned(),
                         author_display_color: color_int_to_hex_string(quote_author.get_color()),
                         override_sender_name: quote.get_override_sender_name(),
@@ -183,7 +191,7 @@ impl MessageObject {
             .map(Into::into)
             .collect();
 
-        Ok(MessageObject {
+        let message_object = MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
             from_id: message.get_from_id().to_u32(),
@@ -239,12 +247,17 @@ impl MessageObject {
             file_name: message.get_filename(),
             webxdc_info,
 
+            // On a WebxdcInfoMessage this might include a hash holding
+            // information about a specific position or state in a webxdc app
+            webxdc_href: message.get_webxdc_href(),
+
             download_state,
 
             reactions,
 
             vcard_contact: vcard_contacts.first().cloned(),
-        })
+        };
+        Ok(Some(message_object))
     }
 }
 
@@ -264,6 +277,9 @@ pub enum MessageViewtype {
     Gif,
 
     /// Message containing a sticker, similar to image.
+    /// NB: When sending, the message viewtype may be changed to `Image` by some heuristics like
+    /// checking for transparent pixels. Use `Message::force_sticker()` to disable them.
+    ///
     /// If possible, the ui should display the image without borders in a transparent way.
     /// A click on a sticker will offer to install the sticker set in some future.
     Sticker,

deltachat-jsonrpc/src/api/types/qr.rs

@@ -6,87 +6,161 @@ use typescript_type_def::TypeDef;
 #[serde(rename = "Qr", rename_all = "camelCase")]
 #[serde(tag = "kind")]
 pub enum QrObject {
+    /// Ask the user whether to verify the contact.
+    ///
+    /// If the user agrees, pass this QR code to [`crate::securejoin::join_securejoin`].
     AskVerifyContact {
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user whether to join the group.
     AskVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Contact fingerprint is verified.
+    ///
+    /// Ask the user if they want to start chatting.
     FprOk {
+        /// Contact ID.
         contact_id: u32,
     },
+    /// Scanned fingerprint does not match the last seen fingerprint.
     FprMismatch {
+        /// Contact ID.
         contact_id: Option<u32>,
     },
+    /// The scanned QR code contains a fingerprint but no e-mail address.
     FprWithoutAddr {
+        /// Key fingerprint.
         fingerprint: String,
     },
+    /// Ask the user if they want to create an account on the given domain.
     Account {
+        /// Server domain name.
         domain: String,
     },
+    /// Provides a backup that can be retrieved using iroh-net based backup transfer protocol.
     Backup2 {
+        /// Authentication token.
         auth_token: String,
-
+        /// Iroh node address.
         node_addr: String,
     },
+    /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
     },
+    /// Ask the user if they want to use the given proxy.
+    ///
+    /// Note that HTTP(S) URLs without a path
+    /// and query parameters are treated as HTTP(S) proxy URL.
+    /// UI may want to still offer to open the URL
+    /// in the browser if QR code contents
+    /// starts with `http://` or `https://`
+    /// and the QR code was not scanned from
+    /// the proxy configuration screen.
     Proxy {
+        /// Proxy URL.
+        ///
+        /// This is the URL that is going to be added.
         url: String,
+        /// Host extracted from the URL to display in the UI.
         host: String,
+        /// Port extracted from the URL to display in the UI.
         port: u16,
     },
+    /// Contact address is scanned.
+    ///
+    /// Optionally, a draft message could be provided.
+    /// Ask the user if they want to start chatting.
     Addr {
+        /// Contact ID.
         contact_id: u32,
+        /// Draft message.
         draft: Option<String>,
     },
-    Url {
-        url: String,
-    },
-    Text {
-        text: String,
-    },
+    /// URL scanned.
+    ///
+    /// Ask the user if they want to open a browser or copy the URL to clipboard.
+    Url { url: String },
+    /// Text scanned.
+    ///
+    /// Ask the user if they want to copy the text to clipboard.
+    Text { text: String },
+    /// Ask the user if they want to withdraw their own QR code.
     WithdrawVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to withdraw their own group invite QR code.
     WithdrawVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own QR code.
     ReviveVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own group invite QR code.
     ReviveVerifyGroup {
+        /// Contact ID.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
-    Login {
-        address: String,
-    },
+    /// `dclogin:` scheme parameters.
+    ///
+    /// Ask the user if they want to login with the email address.
+    Login { address: String },
 }
 
 impl From<Qr> for QrObject {
@@ -141,7 +215,6 @@ impl From<Qr> for QrObject {
                 auth_token,
             } => QrObject::Backup2 {
                 node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
-
                 auth_token,
             },
             Qr::WebrtcInstance {

deltachat-jsonrpc/src/api/types/webxdc.rs

@@ -35,6 +35,14 @@ pub struct WebxdcMessageInfo {
     source_code_url: Option<String>,
     /// True if full internet access should be granted to the app.
     internet_access: bool,
+    /// Address to be used for `window.webxdc.selfAddr` in JS land.
+    self_addr: String,
+    /// Milliseconds to wait before calling `sendUpdate()` again since the last call.
+    /// Should be exposed to `window.sendUpdateInterval` in JS land.
+    send_update_interval: usize,
+    /// Maximum number of bytes accepted for a serialized update object.
+    /// Should be exposed to `window.sendUpdateMaxSize` in JS land.
+    send_update_max_size: usize,
 }
 
 impl WebxdcMessageInfo {
@@ -49,7 +57,11 @@ impl WebxdcMessageInfo {
             document,
             summary,
             source_code_url,
+            request_integration: _,
             internet_access,
+            self_addr,
+            send_update_interval,
+            send_update_max_size,
         } = message.get_webxdc_info(context).await?;
 
         Ok(Self {
@@ -59,6 +71,9 @@ impl WebxdcMessageInfo {
             summary: maybe_empty_string_to_option(summary),
             source_code_url: maybe_empty_string_to_option(source_code_url),
             internet_access,
+            self_addr,
+            send_update_interval,
+            send_update_max_size,
         })
     }
 }

deltachat-jsonrpc/src/lib.rs

@@ -1,4 +1,6 @@
 #![recursion_limit = "256"]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 pub mod api;
 pub use yerpc;
 

deltachat-jsonrpc/typescript/package.json

@@ -58,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.149.0"
+  "version": "1.155.4"
 }

deltachat-ratelimit/src/lib.rs

@@ -90,6 +90,11 @@ impl Ratelimit {
     pub fn until_can_send(&self) -> Duration {
         self.until_can_send_at(SystemTime::now())
     }
+
+    /// Returns minimum possible update interval in milliseconds.
+    pub fn update_interval(&self) -> usize {
+        (self.window.as_millis() as f64 / self.quota) as usize
+    }
 }
 
 #[cfg(test)]
@@ -102,6 +107,7 @@ mod tests {
 
         let mut ratelimit = Ratelimit::new_at(Duration::new(60, 0), 3.0, now);
         assert!(ratelimit.can_send_at(now));
+        assert_eq!(ratelimit.update_interval(), 20_000);
 
         // Send burst of 3 messages.
         ratelimit.send_at(now);

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.149.0"
+version = "1.155.4"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"
@@ -8,12 +8,12 @@ repository = "https://github.com/deltachat/deltachat-core-rust"
 [dependencies]
 anyhow = { workspace = true }
 deltachat = { workspace = true, features = ["internals"]}
-dirs = "5"
+dirs = "6"
 log = { workspace = true }
 nu-ansi-term = { workspace = true }
 qr2term = "0.3.3"
 rusqlite = { workspace = true }
-rustyline = "14"
+rustyline = "15"
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 

deltachat-repl/src/cmdline.rs

@@ -939,7 +939,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             } else {
                 Viewtype::File
             });
-            msg.set_file(arg1, None);
+            msg.set_file_and_deduplicate(&context, Path::new(arg1), None, None)?;
             msg.set_text(arg2.to_string());
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
@@ -969,9 +969,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "Arguments <msg-id> <json status update> expected"
             );
             let msg_id = MsgId::new(arg1.parse()?);
-            context
-                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
-                .await?;
+            context.send_webxdc_status_update(msg_id, arg2).await?;
         }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");

deltachat-repl/src/main.rs

@@ -22,7 +22,7 @@ use log::{error, info, warn};
 use nu_ansi_term::Color;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
-use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
+use rustyline::highlight::{CmdKind as HighlightCmdKind, Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
 use rustyline::validate::Validator;
 use rustyline::{
@@ -298,8 +298,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
-        self.highlighter.highlight_char(line, pos, forced)
+    fn highlight_char(&self, line: &str, pos: usize, kind: HighlightCmdKind) -> bool {
+        self.highlighter.highlight_char(line, pos, kind)
     }
 }
 

deltachat-rpc-client/README.md

@@ -25,7 +25,8 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
+2. Install tox `pip install -U tox`
+3. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.149.0"
+version = "1.155.4"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -13,7 +13,6 @@ classifiers = [
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS :: MacOS X",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -24,9 +23,7 @@ classifiers = [
     "Topic :: Communications :: Email"
 ]
 readme = "README.md"
-dependencies = [
-    "imap-tools",
-]
+requires-python = ">=3.8"
 
 [tool.setuptools.package-data]
 deltachat_rpc_client = [

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -238,6 +238,11 @@ class Chat:
         contacts = self._rpc.get_chat_contacts(self.account.id, self.id)
         return [Contact(self.account, contact_id) for contact_id in contacts]
 
+    def get_past_contacts(self) -> list[Contact]:
+        """Get past contacts for this chat."""
+        past_contacts = self._rpc.get_past_chat_contacts(self.account.id, self.id)
+        return [Contact(self.account, contact_id) for contact_id in past_contacts]
+
     def set_image(self, path: str) -> None:
         """Set profile image of this chat.
 

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -41,6 +41,7 @@ class EventType(str, Enum):
     REACTIONS_CHANGED = "ReactionsChanged"
     INCOMING_MSG = "IncomingMsg"
     INCOMING_MSG_BUNCH = "IncomingMsgBunch"
+    INCOMING_REACTION = "IncomingReaction"
     MSGS_NOTICED = "MsgsNoticed"
     MSG_DELIVERED = "MsgDelivered"
     MSG_FAILED = "MsgFailed"
@@ -61,6 +62,8 @@ class EventType(str, Enum):
     WEBXDC_INSTANCE_DELETED = "WebxdcInstanceDeleted"
     CHATLIST_CHANGED = "ChatlistChanged"
     CHATLIST_ITEM_CHANGED = "ChatlistItemChanged"
+    ACCOUNTS_CHANGED = "AccountsChanged"
+    ACCOUNTS_ITEM_CHANGED = "AccountsItemChanged"
     CONFIG_SYNCED = "ConfigSynced"
     WEBXDC_REALTIME_DATA = "WebxdcRealtimeData"
     WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED = "WebxdcRealtimeAdvertisementReceived"

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -131,10 +131,7 @@ class Rpc:
 
     def reader_loop(self) -> None:
         try:
-            while True:
-                line = self.process.stdout.readline()
-                if not line:  # EOF
-                    break
+            while line := self.process.stdout.readline():
                 response = json.loads(line)
                 if "id" in response:
                     response_id = response["id"]
@@ -150,10 +147,7 @@ class Rpc:
     def writer_loop(self) -> None:
         """Writer loop ensuring only a single thread writes requests."""
         try:
-            while True:
-                request = self.request_queue.get()
-                if not request:
-                    break
+            while request := self.request_queue.get():
                 data = (json.dumps(request) + "\n").encode()
                 self.process.stdin.write(data)
                 self.process.stdin.flush()

deltachat-rpc-client/tests/conftest.py

@@ -1,7 +1,3 @@
-"""
-Internal Python-level IMAP handling used by the tests.
-"""
-
 from __future__ import annotations
 
 import imaplib
@@ -11,17 +7,11 @@ import ssl
 from contextlib import contextmanager
 from typing import TYPE_CHECKING
 
-from imap_tools import (
-    AND,
-    Header,
-    MailBox,
-    MailMessage,
-    MailMessageFlags,
-    errors,
-)
+import pytest
+from imap_tools import AND, Header, MailBox, MailMessage, MailMessageFlags, errors
 
 if TYPE_CHECKING:
-    from . import Account
+    from deltachat_rpc_client import Account
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -29,6 +19,8 @@ ALL = "1:*"
 
 
 class DirectImap:
+    """Internal Python-level IMAP handling."""
+
     def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
@@ -212,3 +204,8 @@ class IdleManager:
     def done(self):
         """send idle-done to server if we are currently in idle mode."""
         return self.direct_imap.conn.idle.stop()
+
+
+@pytest.fixture
+def direct_imap():
+    return DirectImap

deltachat-rpc-client/tests/test_account_events.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from deltachat_rpc_client import EventType
+
+if TYPE_CHECKING:
+    from deltachat_rpc_client.pytestplugin import ACFactory
+
+
+def test_event_on_configuration(acfactory: ACFactory) -> None:
+    """
+    Test if ACCOUNTS_ITEM_CHANGED event is emitted on configure
+    """
+
+    account = acfactory.new_preconfigured_account()
+    account.clear_all_events()
+    assert not account.is_configured()
+    future = account.configure.future()
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.ACCOUNTS_ITEM_CHANGED:
+            break
+    assert account.is_configured()
+
+    future()
+
+
+# other tests are written in rust: src/tests/account_events.rs

deltachat-rpc-client/tests/test_securejoin.py

@@ -73,22 +73,25 @@ def test_qr_securejoin(acfactory, protect, tmp_path):
     qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
-    # Check that at least some of the handshake messages are deleted.
+    # Alice deletes "vg-request".
+    while True:
+        event = alice.wait_for_event()
+        if event["kind"] == "ImapMessageDeleted":
+            break
+    alice.wait_for_securejoin_inviter_success()
+    # Bob deletes "vg-auth-required", Alice deletes "vg-request-with-auth".
     for ac in [alice, bob]:
         while True:
             event = ac.wait_for_event()
             if event["kind"] == "ImapMessageDeleted":
                 break
-
-    alice.wait_for_securejoin_inviter_success()
+    bob.wait_for_securejoin_joiner_success()
 
     # Test that Alice verified Bob's profile.
     alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
     alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
     assert alice_contact_bob_snapshot.is_verified
 
-    bob.wait_for_securejoin_joiner_success()
-
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
     assert snapshot.chat.get_basic_snapshot().is_protected == protect

deltachat-rpc-client/tests/test_something.py

@@ -12,7 +12,6 @@ import pytest
 
 from deltachat_rpc_client import Contact, EventType, Message, events
 from deltachat_rpc_client.const import DownloadState, MessageState
-from deltachat_rpc_client.direct_imap import DirectImap
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -232,7 +231,9 @@ def test_chat(acfactory) -> None:
     group.get_fresh_message_count()
     group.mark_noticed()
     assert group.get_contacts()
-    group.remove_contact(alice_chat_bob)
+    assert group.get_past_contacts() == []
+    group.remove_contact(alice_contact_bob)
+    assert len(group.get_past_contacts()) == 1
     group.get_locations()
 
 
@@ -536,7 +537,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
     assert list(reactions.reactions_by_contact.values())[0] == [react_str]
 
 
-def test_reactions_for_a_reordering_move(acfactory):
+def test_reactions_for_a_reordering_move(acfactory, direct_imap):
     """When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
     their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
     processed by receive_imf in the wrong order, and, particularly, reactions were processed before
@@ -560,7 +561,7 @@ def test_reactions_for_a_reordering_move(acfactory):
     msg1.send_reaction(react_str).wait_until_delivered()
 
     logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
-    ac2_direct_imap = DirectImap(ac2)
+    ac2_direct_imap = direct_imap(ac2)
     ac2_direct_imap.connect()
     for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
         ac2_direct_imap.conn.move(uid, "DeltaChat")

deltachat-rpc-client/tests/test_webxdc.py

@@ -24,6 +24,9 @@ def test_webxdc(acfactory) -> None:
         "name": "Chess Board",
         "sourceCodeUrl": None,
         "summary": None,
+        "selfAddr": webxdc_info["selfAddr"],
+        "sendUpdateInterval": 1000,
+        "sendUpdateMaxSize": 18874368,
     }
 
     status_updates = message.get_webxdc_status_updates()

deltachat-rpc-client/tox.ini

@@ -16,6 +16,7 @@ deps =
     pytest
     pytest-timeout
     pytest-xdist
+    imap-tools
 
 [testenv:lint]
 skipsdist = True

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.149.0"
+version = "1.155.4"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/README.md

@@ -65,13 +65,13 @@ so by default it uses the prebuilds.
   - this will run `update_optional_dependencies_and_version.js` (in the `prepack` script),
     which puts all platform packages into `optionalDependencies` and updates the `version` in `package.json`
 
-## How to build a version you can use localy on your host machine for development
+## How to build a version you can use locally on your host machine for development
 
-You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have seperate scripts for making it work for local installation.
+You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have separate scripts for making it work for local installation.
 
 - If you just need your host platform run `python scripts/make_local_dev_version.py`
 - note: this clears the `platform_package` folder
-- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple plaftorms with `build_platform_package.py`
+- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple platforms with `build_platform_package.py`
 
 ## Thanks to nlnet
 

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.149.0"
+  "version": "1.155.4"
 }

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -6,7 +6,7 @@ const expected_cwd = join(dirname(fileURLToPath(import.meta.url)), "..");
 
 if (process.cwd() !== expected_cwd) {
   console.error(
-    "CWD missmatch: this script needs to be run from " + expected_cwd,
+    "CWD mismatch: this script needs to be run from " + expected_cwd,
     { actual: process.cwd(), expected: expected_cwd }
   );
   process.exit(1);
@@ -40,7 +40,7 @@ const platform_package_names = await Promise.all(
         "has a different version than the version of the rpc server.",
         { rpc_server: version, platform_package: p.version }
       );
-      throw new Error("version missmatch");
+      throw new Error("version mismatch");
     }
     return { folder_name: name, package_name: p.name };
   })

deltachat-rpc-server/src/main.rs

@@ -66,7 +66,7 @@ async fn main_impl() -> Result<()> {
 
     // Logs from `log` crate and traces from `tracing` crate
     // are configurable with `RUST_LOG` environment variable
-    // and go to stderr to avoid interferring with JSON-RPC using stdout.
+    // and go to stderr to avoid interfering with JSON-RPC using stdout.
     tracing_subscriber::fmt()
         .with_env_filter(EnvFilter::from_default_env())
         .with_writer(std::io::stderr)

deny.toml

@@ -12,9 +12,8 @@ ignore = [
     # Unmaintained encoding
     "RUSTSEC-2021-0153",
 
-    # Unmaintained proc-macro-error
-    # <https://rustsec.org/advisories/RUSTSEC-2024-0370>
-    "RUSTSEC-2024-0370",
+    # Unmaintained instant
+    "RUSTSEC-2024-0384",
 ]
 
 [bans]
@@ -27,13 +26,15 @@ skip = [
      { name = "base64", version = "<0.21" },
      { name = "base64", version = "0.21.7" },
      { name = "bitflags", version = "1.3.2" },
+     { name = "core-foundation", version = "0.9.4" },
      { name = "event-listener", version = "2.5.3" },
-     { name = "event-listener", version = "4.0.3" },
-     { name = "fastrand", version = "1.9.0" },
-     { name = "futures-lite", version = "1.13.0" },
+     { name = "generator", version = "0.7.5" },
      { name = "getrandom", version = "<0.2" },
      { name = "http", version = "0.2.12" },
+     { name = "loom", version = "0.5.6" },
+     { name = "netlink-packet-route", version = "0.17.1" },
      { name = "nix", version = "0.26.4" },
+     { name = "nix", version = "0.27.1" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
@@ -41,22 +42,33 @@ skip = [
      { name = "redox_syscall", version = "0.3.5" },
      { name = "regex-automata", version = "0.1.10" },
      { name = "regex-syntax", version = "0.6.29" },
+     { name = "rtnetlink", version = "0.13.1" },
+     { name = "security-framework", version = "2.11.1" },
      { name = "sync_wrapper", version = "0.1.2" },
      { name = "syn", version = "1.0.109" },
+     { name = "thiserror-impl", version = "1.0.69" },
+     { name = "thiserror", version = "1.0.69" },
      { name = "time", version = "<0.3" },
+     { name = "tokio-tungstenite", version = "0.21.0" },
+     { name = "tungstenite", version = "0.21.0" },
+     { name = "unicode-width", version = "0.1.11" },
      { name = "wasi", version = "<0.11" },
-     { name = "windows_aarch64_gnullvm", version = "<0.52" },
-     { name = "windows_aarch64_msvc", version = "<0.52" },
-     { name = "windows-core", version = "<0.54.0" },
-     { name = "windows_i686_gnu", version = "<0.52" },
-     { name = "windows_i686_msvc", version = "<0.52" },
-     { name = "windows-sys", version = "<0.59" },
-     { name = "windows-targets", version = "<0.52" },
-     { name = "windows", version = "<0.54.0" },
-     { name = "windows_x86_64_gnullvm", version = "<0.52" },
-     { name = "windows_x86_64_gnu", version = "<0.52" },
-     { name = "windows_x86_64_msvc", version = "<0.52" },
-     { name = "winreg", version = "0.50.0" },
+     { name = "windows" },
+     { name = "windows_aarch64_gnullvm" },
+     { name = "windows_aarch64_msvc" },
+     { name = "windows-core" },
+     { name = "windows_i686_gnu" },
+     { name = "windows_i686_gnullvm" },
+     { name = "windows_i686_msvc" },
+     { name = "windows-implement" },
+     { name = "windows-interface" },
+     { name = "windows-result" },
+     { name = "windows-strings" },
+     { name = "windows-sys" },
+     { name = "windows-targets" },
+     { name = "windows_x86_64_gnu" },
+     { name = "windows_x86_64_gnullvm" },
+     { name = "windows_x86_64_msvc" },
 ]
 
 
@@ -72,6 +84,7 @@ allow = [
     "MIT",
     "MPL-2.0",
     "OpenSSL",
+    "Unicode-3.0",
     "Unicode-DFS-2016",
     "Zlib",
 ]
@@ -86,6 +99,5 @@ license-files = [
 [sources.allow-org]
 # Organisations which we allow git sources from.
 github = [
-       "async-email",
        "deltachat",
 ]

flake.lock

@@ -7,11 +7,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1729628358,
-        "narHash": "sha256-2HDSc6BL+bE3S1l3Gn0Z8wWvvfBEUEjvXkNIQ11Aifk=",
+        "lastModified": 1731356359,
+        "narHash": "sha256-vYqJnu6jotmWpPT4DgzHVdvNIZcKZCIUqS8QaptsZA0=",
         "owner": "tadfisher",
         "repo": "android-nixpkgs",
-        "rev": "52b9e0c0f9cff887d2bb4932f8be4e062ba0802d",
+        "rev": "c028ead7e88edb2e94cd7c90ee37593f63ae494a",
         "type": "github"
       },
       "original": {
@@ -47,11 +47,11 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1714112748,
-        "narHash": "sha256-jq6Cpf/pQH85p+uTwPPrGG8Ky/zUOTwMJ7mcqc5M4So=",
+        "lastModified": 1737527504,
+        "narHash": "sha256-Z8S5gLPdIYeKwBXDaSxlJ72ZmiilYhu3418h3RSQZA0=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "3ae4b908a795b6a3824d401a0702e11a7157d7e1",
+        "rev": "aa13f23e3e91b95377a693ac655bbc6545ebec0d",
         "type": "github"
       },
       "original": {
@@ -83,11 +83,11 @@
         "systems": "systems_2"
       },
       "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
         "type": "github"
       },
       "original": {
@@ -101,11 +101,11 @@
         "nixpkgs": "nixpkgs_3"
       },
       "locked": {
-        "lastModified": 1713520724,
-        "narHash": "sha256-CO8MmVDmqZX2FovL75pu5BvwhW+Vugc7Q6ze7Hj8heI=",
+        "lastModified": 1721727458,
+        "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=",
         "owner": "nix-community",
         "repo": "naersk",
-        "rev": "c5037590290c6c7dae2e42e7da1e247e54ed2d49",
+        "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11",
         "type": "github"
       },
       "original": {
@@ -116,11 +116,11 @@
     },
     "nix-filter": {
       "locked": {
-        "lastModified": 1710156097,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "lastModified": 1730207686,
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
         "type": "github"
       },
       "original": {
@@ -131,11 +131,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1729413321,
-        "narHash": "sha256-I4tuhRpZFa6Fu6dcH9Dlo5LlH17peT79vx1y1SpeKt0=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "1997e4aa514312c1af7e2bda7fad1644e778ff26",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -147,11 +147,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1713895582,
-        "narHash": "sha256-cfh1hi+6muQMbi9acOlju3V1gl8BEaZBXBR9jQfQi4U=",
+        "lastModified": 1737469691,
+        "narHash": "sha256-nmKOgAU48S41dTPIXAq0AHZSehWUn6ZPrUKijHAMmIk=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "572af610f6151fd41c212f897c71f7056e3fb518",
+        "rev": "9e4d5190a9482a1fb9d18adf0bdb83c6e506eaab",
         "type": "github"
       },
       "original": {
@@ -163,10 +163,9 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1711668574,
-        "narHash": "sha256-u1dfs0ASQIEr1icTVrsKwg2xToIpn7ZXxW3RHfHxshg=",
-        "path": "/nix/store/9fpv0kjq9a80isa1wkkvrdqsh9dpcn05-source",
-        "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659",
+        "lastModified": 0,
+        "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=",
+        "path": "/nix/store/zq2axpgzd5kykk1v446rkffj3bxa2m2h-source",
         "type": "path"
       },
       "original": {
@@ -176,11 +175,11 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1729413321,
-        "narHash": "sha256-I4tuhRpZFa6Fu6dcH9Dlo5LlH17peT79vx1y1SpeKt0=",
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "1997e4aa514312c1af7e2bda7fad1644e778ff26",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -203,11 +202,11 @@
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1714031783,
-        "narHash": "sha256-xS/niQsq1CQPOe4M4jvVPO2cnXS/EIeRG5gIopUbk+Q=",
+        "lastModified": 1737453499,
+        "narHash": "sha256-fa5AJI9mjFU2oVXqdCq2oA2pripAXbHzkUkewJRQpxA=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "56bee2ddafa6177b19c631eedc88d43366553223",
+        "rev": "0b68402d781955d526b80e5d479e9e47addb4075",
         "type": "github"
       },
       "original": {

flake.nix

@@ -88,8 +88,7 @@
         cargoLock = {
           lockFile = ./Cargo.lock;
           outputHashes = {
-            "email-0.0.20" = "sha256-rV4Uzqt2Qdrfi5Ti1r+Si1c2iW1kKyWLwOgLkQ5JGGw=";
-            "encoded-words-0.2.0" = "sha256-KK9st0hLFh4dsrnLd6D8lC6pRFFs8W+WpZSGMGJcosk=";
+            "email-0.0.20" = "sha256-cfR3D5jFQpw32bGsgapK2Uwuxmht+rRK/n1ZUmCb2WA=";
             "lettre-0.9.2" = "sha256-+hU1cFacyyeC9UGVBpS14BWlJjHy90i/3ynMkKAzclk=";
           };
         };
@@ -363,6 +362,8 @@
           mkRustPackages "x86_64-linux" //
           mkRustPackages "armv7l-linux" //
           mkRustPackages "armv6l-linux" //
+          mkRustPackages "x86_64-darwin" //
+          mkRustPackages "aarch64-darwin" //
           mkAndroidPackages "armeabi-v7a" //
           mkAndroidPackages "arm64-v8a" //
           mkAndroidPackages "x86" //
@@ -479,8 +480,8 @@
                   pkgs.python3
                   pkgs.python3Packages.wheel
                 ];
-                buildPhase = ''python3 scripts/wheel-rpc-server.py source deltachat-rpc-server-${manifest.version}.tar.gz'';
-                installPhase = ''mkdir -p $out; cp -av deltachat-rpc-server-${manifest.version}.tar.gz $out'';
+                buildPhase = ''python3 scripts/wheel-rpc-server.py source deltachat_rpc_server-${manifest.version}.tar.gz'';
+                installPhase = ''mkdir -p $out; cp -av deltachat_rpc_server-${manifest.version}.tar.gz $out'';
               };
 
             deltachat-rpc-client =
@@ -533,28 +534,30 @@
               };
           };
 
-        devShells.default = let 
-          pkgs = import nixpkgs {
-            system = system;
-            overlays = [ fenix.overlays.default ];
-          };
-          in pkgs.mkShell {
+        devShells.default =
+          let
+            pkgs = import nixpkgs {
+              system = system;
+              overlays = [ fenix.overlays.default ];
+            };
+          in
+          pkgs.mkShell {
 
-          buildInputs = with pkgs; [
-            (fenix.packages.${system}.complete.withComponents [
-              "cargo"
-              "clippy"
-              "rust-src"
-              "rustc"
-              "rustfmt"
-            ])
-            cargo-deny
-            rust-analyzer-nightly
-            cargo-nextest
-            perl # needed to build vendored OpenSSL
-            git-cliff
-          ];
-        };
+            buildInputs = with pkgs; [
+              (fenix.packages.${system}.complete.withComponents [
+                "cargo"
+                "clippy"
+                "rust-src"
+                "rustc"
+                "rustfmt"
+              ])
+              cargo-deny
+              rust-analyzer-nightly
+              cargo-nextest
+              perl # needed to build vendored OpenSSL
+              git-cliff
+            ];
+          };
       }
     );
 }

format-flowed/src/lib.rs

@@ -8,6 +8,8 @@
 //! is assumed to be set to "no".
 //!
 //! For received messages, DelSp parameter is honoured.
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///

node/constants.js

@@ -1,6 +1,7 @@
 // Generated!
 
 module.exports = {
+  DC_CERTCK_ACCEPT_INVALID: 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES: 3,
   DC_CERTCK_AUTO: 0,
   DC_CERTCK_STRICT: 1,
@@ -30,6 +31,8 @@ module.exports = {
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
+  DC_EVENT_ACCOUNTS_CHANGED: 2302,
+  DC_EVENT_ACCOUNTS_ITEM_CHANGED: 2303,
   DC_EVENT_CHANNEL_OVERFLOW: 2400,
   DC_EVENT_CHATLIST_CHANGED: 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED: 2301,
@@ -51,6 +54,7 @@ module.exports = {
   DC_EVENT_INCOMING_MSG: 2005,
   DC_EVENT_INCOMING_MSG_BUNCH: 2006,
   DC_EVENT_INCOMING_REACTION: 2002,
+  DC_EVENT_INCOMING_WEBXDC_NOTIFY: 2003,
   DC_EVENT_INFO: 100,
   DC_EVENT_LOCATION_CHANGED: 2035,
   DC_EVENT_MSGS_CHANGED: 2000,

node/events.js

@@ -17,6 +17,7 @@ module.exports = {
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
   2002: 'DC_EVENT_INCOMING_REACTION',
+  2003: 'DC_EVENT_INCOMING_WEBXDC_NOTIFY',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -43,5 +44,7 @@ module.exports = {
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2302: 'DC_EVENT_ACCOUNTS_CHANGED',
+  2303: 'DC_EVENT_ACCOUNTS_ITEM_CHANGED',
   2400: 'DC_EVENT_CHANNEL_OVERFLOW'
 }

node/lib/constants.ts

@@ -1,6 +1,7 @@
 // Generated!
 
 export enum C {
+  DC_CERTCK_ACCEPT_INVALID = 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3,
   DC_CERTCK_AUTO = 0,
   DC_CERTCK_STRICT = 1,
@@ -30,6 +31,8 @@ export enum C {
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
+  DC_EVENT_ACCOUNTS_CHANGED = 2302,
+  DC_EVENT_ACCOUNTS_ITEM_CHANGED = 2303,
   DC_EVENT_CHANNEL_OVERFLOW = 2400,
   DC_EVENT_CHATLIST_CHANGED = 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED = 2301,
@@ -51,6 +54,7 @@ export enum C {
   DC_EVENT_INCOMING_MSG = 2005,
   DC_EVENT_INCOMING_MSG_BUNCH = 2006,
   DC_EVENT_INCOMING_REACTION = 2002,
+  DC_EVENT_INCOMING_WEBXDC_NOTIFY = 2003,
   DC_EVENT_INFO = 100,
   DC_EVENT_LOCATION_CHANGED = 2035,
   DC_EVENT_MSGS_CHANGED = 2000,
@@ -324,6 +328,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
   2002: 'DC_EVENT_INCOMING_REACTION',
+  2003: 'DC_EVENT_INCOMING_WEBXDC_NOTIFY',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -350,5 +355,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2302: 'DC_EVENT_ACCOUNTS_CHANGED',
+  2303: 'DC_EVENT_ACCOUNTS_ITEM_CHANGED',
   2400: 'DC_EVENT_CHANNEL_OVERFLOW',
 }

node/lib/message.ts

@@ -299,10 +299,6 @@ export class Message {
     return Boolean(binding.dcn_msg_is_forwarded(this.dc_msg))
   }
 
-  isIncreation() {
-    return Boolean(binding.dcn_msg_is_increation(this.dc_msg))
-  }
-
   isInfo() {
     return Boolean(binding.dcn_msg_is_info(this.dc_msg))
   }

node/scripts/rebuild-core.js

@@ -9,7 +9,7 @@ const buildArgs = [
   'build',
   '--release',
   '--features',
-  'vendored,jsonrpc',
+  'vendored',
   '-p',
   'deltachat_ffi'
 ]

node/src/module.c

@@ -2374,17 +2374,6 @@ NAPI_METHOD(dcn_msg_is_forwarded) {
   NAPI_RETURN_INT32(is_forwarded);
 }
 
-NAPI_METHOD(dcn_msg_is_increation) {
-  NAPI_ARGV(1);
-  NAPI_DC_MSG();
-
-  //TRACE("calling..");
-  int is_increation = dc_msg_is_increation(dc_msg);
-  //TRACE("result %d", is_increation);
-
-  NAPI_RETURN_INT32(is_increation);
-}
-
 NAPI_METHOD(dcn_msg_is_info) {
   NAPI_ARGV(1);
   NAPI_DC_MSG();
@@ -3555,7 +3544,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_msg_has_location);
   NAPI_EXPORT_FUNCTION(dcn_msg_has_html);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_forwarded);
-  NAPI_EXPORT_FUNCTION(dcn_msg_is_increation);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_info);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_sent);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_setupmessage);

node/test/test.mjs

@@ -8,7 +8,7 @@ import { EventId2EventName, C } from '../dist/constants.js'
 import { join } from 'path'
 import { statSync } from 'fs'
 import { Context } from '../dist/context.js'
-import {fileURLToPath} from 'url';
+import { fileURLToPath } from 'url';
 
 const __dirname = fileURLToPath(new URL('.', import.meta.url));
 
@@ -444,7 +444,7 @@ describe('Offline Tests with unconfigured account', function () {
     context.setChatProfileImage(chatId, imagePath)
     const blobPath = context.getChat(chatId).getProfileImage()
     expect(blobPath.startsWith(blobs)).to.be.true
-    expect(blobPath.includes('image')).to.be.true
+    expect(blobPath.includes('image')).to.be.false
     expect(blobPath.endsWith('.jpeg')).to.be.true
 
     context.setChatProfileImage(chatId, null)
@@ -536,7 +536,6 @@ describe('Offline Tests with unconfigured account', function () {
     strictEqual(msg.getWidth(), 0, 'no message width')
     strictEqual(msg.isDeadDrop(), false, 'not deaddrop')
     strictEqual(msg.isForwarded(), false, 'not forwarded')
-    strictEqual(msg.isIncreation(), false, 'not in creation')
     strictEqual(msg.isInfo(), false, 'not an info message')
     strictEqual(msg.isSent(), false, 'messge is not sent')
     strictEqual(msg.isSetupmessage(), false, 'not an autocrypt setup message')

package.json

@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.149.0"
+  "version": "1.155.4"
 }

python/doc/cffi/install.rst

@@ -52,10 +52,7 @@ python3-venv` should give you a usable python installation.
 
 First, build the core library::
 
-   cargo build --release -p deltachat_ffi --features jsonrpc
-
-`jsonrpc` feature is required even if not used by the bindings
-because `deltachat.h` includes JSON-RPC functions unconditionally.
+   cargo build --release -p deltachat_ffi
 
 Create the virtual environment and activate it::
 

python/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.149.0"
+version = "1.155.4"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 authors = [
     { name = "holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors" },
 ]

python/src/deltachat/account.py

@@ -671,9 +671,6 @@ class Account:
     def get_connectivity_html(self) -> str:
         return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
 
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """start this account's IO scheduling (Rust-core async scheduler).
 

python/src/deltachat/chat.py

@@ -271,8 +271,7 @@ class Chat:
 
         :param msg: a :class:`deltachat.message.Message` instance
            previously returned by
-           e.g. :meth:`deltachat.message.Message.new_empty` or
-           :meth:`prepare_file`.
+           e.g. :meth:`deltachat.message.Message.new_empty`.
         :raises ValueError: if message can not be sent.
 
         :returns: a :class:`deltachat.message.Message` instance as
@@ -341,37 +340,6 @@ class Chat:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
 
-    def prepare_message(self, msg):
-        """prepare a message for sending.
-
-        :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
-        """
-        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if msg_id == 0:
-            raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
-
-    def prepare_message_file(self, path, mime_type=None, view_type="file"):
-        """prepare a message for sending and return the resulting Message instance.
-
-        To actually send the message, call :meth:`send_prepared`.
-        The file must be inside the blob directory.
-
-        :param path: path to the file.
-        :param mime_type: the mime-type of this file, defaults to auto-detection.
-        :param view_type: "text", "image", "gif", "audio", "video", "file"
-        :raises ValueError: if message can not be prepared/chat does not exist.
-        :returns: the resulting :class:`Message` instance
-        """
-        msg = Message.new_empty(self.account, view_type)
-        msg.set_file(path, mime_type)
-        return self.prepare_message(msg)
-
     def send_prepared(self, message):
         """send a previously prepared message.
 

python/src/deltachat/events.py

@@ -158,12 +158,6 @@ class FFIEventTracker:
 
             self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
 
-    def wait_for_all_work_done(self):
-        while True:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile(f"(?:{event_name_regex}).*")

python/src/deltachat/message.py

@@ -108,7 +108,9 @@ class Message:
 
     @props.with_doc
     def filename(self):
-        """filename if there was an attachment, otherwise empty string."""
+        """file path if there was an attachment, otherwise empty string.
+        If you want to get the file extension or a user-visible string,
+        use `basename` instead."""
         return from_dc_charpointer(lib.dc_msg_get_file(self._dc_msg))
 
     def set_file(self, path, mime_type=None):
@@ -120,7 +122,8 @@ class Message:
 
     @props.with_doc
     def basename(self) -> str:
-        """basename of the attachment if it exists, otherwise empty string."""
+        """The user-visible name of the attachment (incl. extension)
+        if it exists, otherwise empty string."""
         # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 

python/src/deltachat/testplugin.py

@@ -152,7 +152,7 @@ class TestProcess:
 
     def get_liveconfig_producer(self):
         """provide live account configs, cached on a per-test-process scope
-        so that test functions can re-use already known live configs.
+        so that test functions can reuse already known live configs.
         """
         chatmail_opt = self.pytestconfig.getoption("--chatmail")
         if chatmail_opt:

python/tests/test_1_online.py

@@ -181,15 +181,16 @@ def test_send_file_twice_unicode_filename_mangling(tmp_path, acfactory, lp):
     msg = send_and_receive_message()
     assert msg.text == "withfile"
     assert open(msg.filename).read() == "some data"
-    msg.filename.index(basename)
-    assert msg.filename.endswith(ext)
+    msg.basename.index(basename)
+    assert msg.basename.endswith(ext)
 
     msg2 = send_and_receive_message()
     assert msg2.text == "withfile"
     assert open(msg2.filename).read() == "some data"
-    msg2.filename.index(basename)
-    assert msg2.filename.endswith(ext)
-    assert msg.filename != msg2.filename
+    msg2.basename.index(basename)
+    assert msg2.basename.endswith(ext)
+    assert msg.filename == msg2.filename  # The file is deduplicated
+    assert msg.basename == msg2.basename
 
 
 def test_send_file_html_attachment(tmp_path, acfactory, lp):
@@ -214,8 +215,8 @@ def test_send_file_html_attachment(tmp_path, acfactory, lp):
     msg = ac2.get_message_by_id(ev.data2)
 
     assert open(msg.filename).read() == content
-    msg.filename.index(basename)
-    assert msg.filename.endswith(ext)
+    msg.basename.index(basename)
+    assert msg.basename.endswith(ext)
 
 
 def test_html_message(acfactory, lp):
@@ -1253,7 +1254,10 @@ def test_no_old_msg_is_fresh(acfactory, lp):
 
 def test_prefer_encrypt(acfactory, lp):
     """Test quorum rule for encryption preference in 1:1 and group chat."""
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+    ac1 = acfactory.new_online_configuring_account(fix_is_chatmail=True)
+    ac2 = acfactory.new_online_configuring_account(fix_is_chatmail=True)
+    ac3 = acfactory.new_online_configuring_account(fix_is_chatmail=True)
+    acfactory.bring_accounts_online()
     ac1.set_config("e2ee_enabled", "0")
     ac2.set_config("e2ee_enabled", "1")
     ac3.set_config("e2ee_enabled", "0")
@@ -1276,7 +1280,8 @@ def test_prefer_encrypt(acfactory, lp):
     lp.sec("ac2: sending message to ac1")
     chat2 = ac2.create_chat(ac1)
     msg2 = chat2.send_text("message2")
-    assert not msg2.is_encrypted()
+    # Own preference is `Mutual` and we have the peer's key.
+    assert msg2.is_encrypted()
     ac1._evtracker.wait_next_incoming_message()
 
     lp.sec("ac1: sending message to group chat with ac2 and ac3")
@@ -1292,8 +1297,8 @@ def test_prefer_encrypt(acfactory, lp):
     ac3.set_config("e2ee_enabled", "1")
     chat3 = ac3.create_chat(ac1)
     msg4 = chat3.send_text("message4")
-    # ac1 still does not prefer encryption
-    assert not msg4.is_encrypted()
+    # Own preference is `Mutual` and we have the peer's key.
+    assert msg4.is_encrypted()
     ac1._evtracker.wait_next_incoming_message()
 
     lp.sec("ac1: sending another message to group chat with ac2 and ac3")
@@ -1366,10 +1371,9 @@ def test_quote_encrypted(acfactory, lp):
         msg_draft.quote = quoted_msg
         chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
+        # Get the draft and send it.
         msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
-        chat.send_prepared(msg_out)
+        chat.send_msg(msg_draft)
 
         chat.set_draft(None)
         assert chat.get_draft() is None
@@ -1899,10 +1903,11 @@ def test_connectivity(acfactory, lp):
 
     ac1.start_io()
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_CONNECTING)
-    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_CONNECTING, dc.const.DC_CONNECTIVITY_CONNECTED)
+    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_CONNECTING, dc.const.DC_CONNECTIVITY_WORKING)
+    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_WORKING, dc.const.DC_CONNECTIVITY_CONNECTED)
 
     lp.sec(
-        "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, "
+        "Test that after calling start_io(), maybe_network() and waiting for `DC_CONNECTIVITY_CONNECTED`, "
         "all messages are fetched",
     )
 
@@ -1911,7 +1916,7 @@ def test_connectivity(acfactory, lp):
         ac2.create_chat(ac1).send_text("Hi")
         idle1.wait_for_new_message()
     ac1.maybe_network()
-    ac1._evtracker.wait_for_all_work_done()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_CONNECTED)
     msgs = ac1.create_chat(ac2).get_messages()
     assert len(msgs) == 1
     assert msgs[0].text == "Hi"
@@ -1927,30 +1932,6 @@ def test_connectivity(acfactory, lp):
     assert len(msgs) == 2
     assert msgs[1].text == "Hi 2"
 
-    lp.sec("Test that the connectivity doesn't flicker to WORKING if there are no new messages")
-
-    ac1.maybe_network()
-    while 1:
-        assert ac1.get_connectivity() == dc.const.DC_CONNECTIVITY_CONNECTED
-        if ac1.all_work_done():
-            break
-        ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    lp.sec("Test that the connectivity doesn't flicker to WORKING if the sender of the message is blocked")
-    ac1.create_contact(ac2).block()
-
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        ac2.create_chat(ac1).send_text("Hi")
-        idle1.wait_for_new_message()
-    ac1.maybe_network()
-
-    while 1:
-        assert ac1.get_connectivity() == dc.const.DC_CONNECTIVITY_CONNECTED
-        if ac1.all_work_done():
-            break
-        ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     lp.sec("Test that the connectivity is NOT_CONNECTED if the password is wrong")
 
     ac1.set_config("configured_mail_pw", "abc")
@@ -1961,32 +1942,6 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
-def test_all_work_done(acfactory, lp):
-    """
-    Tests that calling start_io() immediately followed by maybe_network()
-    and then waiting for all_work_done() reliably fetches the messages
-    delivered while account was offline.
-    In other words, connectivity should not change to a state
-    where all_work_done() returns true until IMAP connection goes idle.
-    """
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    ac1.stop_io()
-    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
-
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        ac2.create_chat(ac1).send_text("Hi")
-        idle1.wait_for_new_message()
-
-    ac1.start_io()
-    ac1.maybe_network()
-    ac1._evtracker.wait_for_all_work_done()
-    msgs = ac1.create_chat(ac2).get_messages()
-    assert len(msgs) == 1
-    assert msgs[0].text == "Hi"
-
-
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.
@@ -2028,7 +1983,7 @@ def test_fetch_deleted_msg(acfactory, lp):
         if ev.name == "DC_EVENT_MSGS_CHANGED":
             pytest.fail("A deleted message was shown to the user")
 
-        if ev.name == "DC_EVENT_INFO" and "INBOX: Idle entering wait-on-remote state" in ev.data2:
+        if ev.name == "DC_EVENT_INFO" and 'IDLE entering wait-on-remote state in folder "INBOX".' in ev.data2:
             break  # DC is done with reading messages
 
 
@@ -2340,9 +2295,8 @@ def test_group_quote(acfactory, lp):
     reply_msg = Message.new_empty(msg.chat.account, "text")
     reply_msg.set_text("reply")
     reply_msg.quote = msg
-    reply_msg = msg.chat.prepare_message(reply_msg)
     assert reply_msg.quoted_text == "hello"
-    msg.chat.send_prepared(reply_msg)
+    msg.chat.send_msg(reply_msg)
 
     lp.sec("ac3: receiving reply")
     received_reply = ac3._evtracker.wait_next_incoming_message()

python/tests/test_2_increation.py

@@ -1,107 +0,0 @@
-import os.path
-import shutil
-from filecmp import cmp
-
-import pytest
-
-
-def wait_msg_delivered(account, msg_list):
-    """wait for one or more MSG_DELIVERED events to match msg_list contents."""
-    msg_list = list(msg_list)
-    while msg_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        msg_list.remove((ev.data1, ev.data2))
-
-
-def wait_msgs_changed(account, msgs_list):
-    """wait for one or more MSGS_CHANGED events to match msgs_list contents."""
-    account.log(f"waiting for msgs_list={msgs_list}")
-    msgs_list = list(msgs_list)
-    while msgs_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        for i, (data1, data2) in enumerate(msgs_list):
-            if ev.data1 == data1:
-                if data2 is None or ev.data2 == data2:
-                    del msgs_list[i]
-                    break
-        else:
-            account.log(f"waiting mismatch data1={data1} data2={data2}")
-    return ev.data2
-
-
-class TestOnlineInCreation:
-    def test_increation_not_blobdir(self, tmp_path, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating in-creation file outside of blobdir")
-        assert str(tmp_path) != ac1.get_blobdir()
-        src = tmp_path / "file.txt"
-        src.touch()
-        with pytest.raises(Exception):
-            chat.prepare_message_file(str(src))
-
-    def test_no_increation_copies_to_blobdir(self, tmp_path, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating file outside of blobdir")
-        assert str(tmp_path) != ac1.get_blobdir()
-        src = tmp_path / "file.txt"
-        src.write_text("hello there\n")
-        msg = chat.send_file(str(src))
-        assert msg.filename.startswith(os.path.join(ac1.get_blobdir(), "file"))
-        assert msg.filename.endswith(".txt")
-
-    def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-
-        chat = ac1.create_chat(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
-
-        lp.sec("create a message with a file in creation")
-        orig = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), "d.png")
-        with open(path, "x") as fp:
-            fp.write("preparing")
-        prepared_original = chat.prepare_message_file(path)
-        assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
-
-        lp.sec("create a new group")
-        chat2 = ac1.create_group_chat("newgroup")
-        wait_msgs_changed(ac1, [(0, 0)])
-
-        lp.sec("add a contact to new group")
-        chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(chat2.id, None)])
-
-        lp.sec("forward the message while still in creation")
-        ac1.forward_messages([prepared_original], chat2)
-        forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
-        forwarded_msg = ac1.get_message_by_id(forwarded_id)
-        assert forwarded_msg.is_out_preparing()
-
-        lp.sec("finish creating the file and send it")
-        assert prepared_original.is_out_preparing()
-        shutil.copyfile(orig, path)
-        chat.send_prepared(prepared_original)
-        assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
-
-        lp.sec("check that both forwarded and original message are proper.")
-        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
-        fwd_msg = ac1.get_message_by_id(forwarded_id)
-        assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
-
-        lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
-        lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
-        assert cmp(received_original.filename, orig, shallow=False)
-
-        lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
-        assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_3_offline.py

@@ -378,30 +378,6 @@ class TestOfflineChat:
         with pytest.raises(ValueError):
             chat1.send_text("msg1")
 
-    def test_prepare_message_and_send(self, ac1, chat1):
-        msg = chat1.prepare_message(Message.new_empty(chat1.account, "text"))
-        msg.set_text("hello world")
-        assert msg.text == "hello world"
-        assert msg.id > 0
-        chat1.send_prepared(msg)
-        assert "Sent" in msg.get_message_info()
-        str(msg)
-        repr(msg)
-        assert msg == ac1.get_message_by_id(msg.id)
-
-    def test_prepare_file(self, ac1, chat1):
-        blobdir = ac1.get_blobdir()
-        p = os.path.join(blobdir, "somedata.txt")
-        with open(p, "w") as f:
-            f.write("some data")
-        message = chat1.prepare_message_file(p)
-        assert message.id > 0
-        message.set_text("hello world")
-        assert message.is_out_preparing()
-        assert message.text == "hello world"
-        chat1.send_prepared(message)
-        assert "Sent" in message.get_message_info()
-
     def test_message_eq_contains(self, chat1):
         msg = chat1.send_text("msg1")
         msg2 = None
@@ -691,8 +667,7 @@ class TestOfflineChat:
         assert os.path.exists(messages[1].filename)
 
     def test_set_get_draft(self, chat1):
-        msg = Message.new_empty(chat1.account, "text")
-        msg1 = chat1.prepare_message(msg)
+        msg1 = Message.new_empty(chat1.account, "text")
         msg1.set_text("hello")
         chat1.set_draft(msg1)
         msg1.set_text("obsolete")
@@ -705,27 +680,12 @@ class TestOfflineChat:
         ac1 = acfactory.get_pseudo_configured_account()
         ac2 = acfactory.get_pseudo_configured_account()
         qr = ac1.get_setup_contact_qr()
-        assert qr.startswith("OPENPGP4FPR:")
+        assert qr.startswith("https://i.delta.chat")
         res = ac2.check_qr(qr)
         assert res.is_ask_verifycontact()
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
-    def test_quote(self, chat1):
-        """Offline quoting test"""
-        msg = Message.new_empty(chat1.account, "text")
-        msg.set_text("Multi\nline\nmessage")
-        assert msg.quoted_text is None
-
-        # Prepare message to assign it a Message-Id.
-        # Messages without Message-Id cannot be quoted.
-        msg = chat1.prepare_message(msg)
-
-        reply_msg = Message.new_empty(chat1.account, "text")
-        reply_msg.set_text("reply")
-        reply_msg.quote = msg
-        assert reply_msg.quoted_text == "Multi\nline\nmessage"
-
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")

python/tests/test_4_lowlevel.py

@@ -212,8 +212,13 @@ def test_logged_ac_process_ffi_failure(acfactory):
             0 / 0
 
     cap = Queue()
-    ac1.log = cap.put
+
+    # Make sure the next attempt to log an event fails.
     ac1.add_account_plugin(FailPlugin())
+
+    # Start capturing events.
+    ac1.log = cap.put
+
     # cause any event eg contact added/changed
     ac1.create_contact("something@example.org")
     res = cap.get(timeout=10)

release-date.in

@@ -1 +1 @@
-2024-11-05
+2025-02-10

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.82.0
+RUST_VERSION=1.84.1
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/make-python-testenv.sh

@@ -11,7 +11,7 @@ set -euo pipefail
 
 export DCC_RS_TARGET=debug
 export DCC_RS_DEV="$PWD"
-cargo build -p deltachat_ffi --features jsonrpc
+cargo build -p deltachat_ffi
 
 tox -c python -e py --devenv venv
 venv/bin/pip install --upgrade pip

scripts/run-python-test.sh

@@ -12,7 +12,7 @@ export DCC_RS_DEV=`pwd`
 
 cd python
 
-cargo build -p deltachat_ffi --features jsonrpc
+cargo build -p deltachat_ffi
 
 # remove and inhibit writing PYC files 
 rm -rf tests/__pycache__

scripts/run_all.sh

@@ -8,7 +8,7 @@ set -e -x
 
 # compile core lib
 
-cargo build --release -p deltachat_ffi --features jsonrpc
+cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV="$PWD"
@@ -31,6 +31,6 @@ unset CHATMAIL_DOMAIN
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,py313,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
+tox --workdir "$TOXWORKDIR" -e py38,py39,py310,py311,py312,py313,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"

scripts/wheel-rpc-server.py

@@ -25,7 +25,7 @@ def build_source_package(version, filename):
 
         def pack(name, contents):
             contents = contents.encode()
-            tar_info = tarfile.TarInfo(f"deltachat-rpc-server-{version}/{name}")
+            tar_info = tarfile.TarInfo(f"deltachat_rpc_server-{version}/{name}")
             tar_info.mode = 0o644
             tar_info.size = len(contents)
             pkg.addfile(tar_info, BytesIO(contents))
@@ -167,7 +167,7 @@ def main():
         cargo_manifest = tomllib.load(fp)
     version = cargo_manifest["package"]["version"]
     if sys.argv[1] == "source":
-        filename = f"deltachat-rpc-server-{version}.tar.gz"
+        filename = f"deltachat_rpc_server-{version}.tar.gz"
         build_source_package(version, filename)
     else:
         arch = sys.argv[1]

src/accounts.rs

@@ -139,6 +139,7 @@ impl Accounts {
         ctx.open("".to_string()).await?;
 
         self.accounts.insert(account_config.id, ctx);
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(account_config.id)
     }
@@ -156,6 +157,7 @@ impl Accounts {
             .build()
             .await?;
         self.accounts.insert(account_config.id, ctx);
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(account_config.id)
     }
@@ -190,6 +192,7 @@ impl Accounts {
                 .context("failed to remove account data")?;
         }
         self.config.remove_account(id).await?;
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(())
     }

src/authres.rs

@@ -260,7 +260,6 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 

src/blob/blob_tests.rs

@@ -0,0 +1,810 @@
+use std::time::Duration;
+
+use super::*;
+use crate::message::{Message, Viewtype};
+use crate::sql;
+use crate::test_utils::{self, TestContext};
+use crate::tools::SystemTime;
+
+fn check_image_size(path: impl AsRef<Path>, width: u32, height: u32) -> image::DynamicImage {
+    tokio::task::block_in_place(move || {
+        let img = ImageReader::open(path)
+            .expect("failed to open image")
+            .with_guessed_format()
+            .expect("failed to guess format")
+            .decode()
+            .expect("failed to decode image");
+        assert_eq!(img.width(), width, "invalid width");
+        assert_eq!(img.height(), height, "invalid height");
+        img
+    })
+}
+
+const FILE_BYTES: &[u8] = b"hello";
+const FILE_DEDUPLICATED: &str = "ea8f163db38682925e4491c5e58d4bb.txt";
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create() {
+    let t = TestContext::new().await;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
+    let fname = t.get_blobdir().join(FILE_DEDUPLICATED);
+    let data = fs::read(fname).await.unwrap();
+    assert_eq!(data, FILE_BYTES);
+    assert_eq!(blob.as_name(), format!("$BLOBDIR/{FILE_DEDUPLICATED}"));
+    assert_eq!(blob.to_abs_path(), t.get_blobdir().join(FILE_DEDUPLICATED));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_lowercase_ext() {
+    let t = TestContext::new().await;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.TXT").unwrap();
+    assert!(
+        blob.as_name().ends_with(".txt"),
+        "Blob {blob:?} should end with .txt"
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_as_file_name() {
+    let t = TestContext::new().await;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
+    assert_eq!(blob.as_file_name(), FILE_DEDUPLICATED);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_as_rel_path() {
+    let t = TestContext::new().await;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
+    assert_eq!(blob.as_rel_path(), Path::new(FILE_DEDUPLICATED));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_suffix() {
+    let t = TestContext::new().await;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
+    assert_eq!(blob.suffix(), Some("txt"));
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "bar").unwrap();
+    assert_eq!(blob.suffix(), None);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_dup() {
+    let t = TestContext::new().await;
+    BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.txt").unwrap();
+    let foo_path = t.get_blobdir().join(FILE_DEDUPLICATED);
+    assert!(foo_path.exists());
+    BlobObject::create_and_deduplicate_from_bytes(&t, b"world", "foo.txt").unwrap();
+    let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+    while let Ok(Some(dirent)) = dir.next_entry().await {
+        let fname = dirent.file_name();
+        if fname == foo_path.file_name().unwrap() {
+            assert_eq!(fs::read(&foo_path).await.unwrap(), FILE_BYTES);
+        } else {
+            let name = fname.to_str().unwrap();
+            assert!(name.ends_with(".txt"));
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_double_ext() {
+    let t = TestContext::new().await;
+    BlobObject::create_and_deduplicate_from_bytes(&t, FILE_BYTES, "foo.tar.gz").unwrap();
+    let foo_path = t.get_blobdir().join(FILE_DEDUPLICATED).with_extension("gz");
+    assert!(foo_path.exists());
+    BlobObject::create_and_deduplicate_from_bytes(&t, b"world", "foo.tar.gz").unwrap();
+    let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
+    while let Ok(Some(dirent)) = dir.next_entry().await {
+        let fname = dirent.file_name();
+        if fname == foo_path.file_name().unwrap() {
+            assert_eq!(fs::read(&foo_path).await.unwrap(), FILE_BYTES);
+        } else {
+            let name = fname.to_str().unwrap();
+            println!("{name}");
+            assert_eq!(name.starts_with("foo"), false);
+            assert_eq!(name.ends_with(".tar.gz"), false);
+            assert!(name.ends_with(".gz"));
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_long_names() {
+    let t = TestContext::new().await;
+    let s = format!("file.{}", "a".repeat(100));
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, b"data", &s).unwrap();
+    let blobname = blob.as_name().split('/').last().unwrap();
+    assert!(blobname.len() < 70);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_and_copy() {
+    let t = TestContext::new().await;
+    let src = t.dir.path().join("src");
+    fs::write(&src, b"boo").await.unwrap();
+    let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
+    assert_eq!(blob.as_name(), "$BLOBDIR/src");
+    let data = fs::read(blob.to_abs_path()).await.unwrap();
+    assert_eq!(data, b"boo");
+
+    let whoops = t.dir.path().join("whoops");
+    assert!(BlobObject::create_and_copy(&t, whoops.as_ref())
+        .await
+        .is_err());
+    let whoops = t.get_blobdir().join("whoops");
+    assert!(!whoops.exists());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_from_path() {
+    let t = TestContext::new().await;
+
+    let src_ext = t.dir.path().join("external");
+    fs::write(&src_ext, b"boo").await.unwrap();
+    let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+        .await
+        .unwrap();
+    assert_eq!(blob.as_name(), "$BLOBDIR/external");
+    let data = fs::read(blob.to_abs_path()).await.unwrap();
+    assert_eq!(data, b"boo");
+
+    let src_int = t.get_blobdir().join("internal");
+    fs::write(&src_int, b"boo").await.unwrap();
+    let blob = BlobObject::new_from_path(&t, &src_int).await.unwrap();
+    assert_eq!(blob.as_name(), "$BLOBDIR/internal");
+    let data = fs::read(blob.to_abs_path()).await.unwrap();
+    assert_eq!(data, b"boo");
+}
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_from_name_long() {
+    let t = TestContext::new().await;
+    let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
+    fs::write(&src_ext, b"boo").await.unwrap();
+    let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
+        .await
+        .unwrap();
+    assert_eq!(
+        blob.as_name(),
+        "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+    );
+}
+
+#[test]
+fn test_is_blob_name() {
+    assert!(BlobObject::is_acceptible_blob_name("foo"));
+    assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
+    assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+    assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
+    assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
+    assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
+}
+
+#[test]
+fn test_sanitise_name() {
+    let (stem, ext) = BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
+    assert_eq!(ext, ".txt");
+    assert!(!stem.is_empty());
+
+    // the extensions are kept together as between stem and extension a number may be added -
+    // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+    let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+    assert_eq!(stem, "wot");
+    assert_eq!(ext, ".tar.gz");
+
+    let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+    assert_eq!(stem, "");
+    assert_eq!(ext, ".foo.bar");
+
+    let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+    assert!(stem.contains("foo"));
+    assert!(!stem.contains('?'));
+    assert_eq!(ext, ".bar");
+
+    let (stem, ext) = BlobObject::sanitise_name("no-extension");
+    assert_eq!(stem, "no-extension");
+    assert_eq!(ext, "");
+
+    let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+    assert_eq!(ext, ".c");
+    assert!(!stem.contains("path"));
+    assert!(!stem.contains("ignored"));
+    assert!(stem.contains("this"));
+    assert!(stem.contains("forbidden"));
+    assert!(!stem.contains('/'));
+    assert!(!stem.contains('\\'));
+    assert!(!stem.contains(':'));
+    assert!(!stem.contains('*'));
+    assert!(!stem.contains('?'));
+
+    let (stem, ext) = BlobObject::sanitise_name(
+        "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz",
+    );
+    assert_eq!(
+        stem,
+        "file.with_lots_of_characters_behind_point_and_double_ending"
+    );
+    assert_eq!(ext, ".tar.gz");
+
+    let (stem, ext) = BlobObject::sanitise_name("a. tar.tar.gz");
+    assert_eq!(stem, "a. tar");
+    assert_eq!(ext, ".tar.gz");
+
+    let (stem, ext) = BlobObject::sanitise_name("Guia_uso_GNB (v0.8).pdf");
+    assert_eq!(stem, "Guia_uso_GNB (v0.8)");
+    assert_eq!(ext, ".pdf");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_add_white_bg() {
+    let t = TestContext::new().await;
+    let bytes0 = include_bytes!("../../test-data/image/logo.png").as_slice();
+    let bytes1 = include_bytes!("../../test-data/image/avatar900x900.png").as_slice();
+    for (bytes, color) in [
+        (bytes0, [255u8, 255, 255, 255]),
+        (bytes1, [253u8, 198, 0, 255]),
+    ] {
+        let avatar_src = t.dir.path().join("avatar.png");
+        fs::write(&avatar_src, bytes).await.unwrap();
+
+        let mut blob = BlobObject::new_from_path(&t, &avatar_src).await.unwrap();
+        let img_wh = 128;
+        let maybe_sticker = &mut false;
+        let strict_limits = true;
+        blob.recode_to_size(&t, None, maybe_sticker, img_wh, 20_000, strict_limits)
+            .unwrap();
+        tokio::task::block_in_place(move || {
+            let img = ImageReader::open(blob.to_abs_path())
+                .unwrap()
+                .with_guessed_format()
+                .unwrap()
+                .decode()
+                .unwrap();
+            assert!(img.width() == img_wh);
+            assert!(img.height() == img_wh);
+            assert_eq!(img.get_pixel(0, 0), Rgba(color));
+        });
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_outside_blobdir() {
+    async fn file_size(path_buf: &Path) -> u64 {
+        fs::metadata(path_buf).await.unwrap().len()
+    }
+
+    let t = TestContext::new().await;
+    let avatar_src = t.dir.path().join("avatar.jpg");
+    let avatar_bytes = include_bytes!("../../test-data/image/avatar1000x1000.jpg");
+    fs::write(&avatar_src, avatar_bytes).await.unwrap();
+    t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+        .await
+        .unwrap();
+    let avatar_blob = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
+    let avatar_path = Path::new(&avatar_blob);
+    assert!(
+        avatar_blob.ends_with("d98cd30ed8f2129bf3968420208849d.jpg"),
+        "The avatar filename should be its hash, put instead it's {avatar_blob}"
+    );
+    let scaled_avatar_size = file_size(avatar_path).await;
+    assert!(scaled_avatar_size < avatar_bytes.len() as u64);
+
+    check_image_size(avatar_src, 1000, 1000);
+    check_image_size(
+        &avatar_blob,
+        constants::BALANCED_AVATAR_SIZE,
+        constants::BALANCED_AVATAR_SIZE,
+    );
+
+    let mut blob = BlobObject::new_from_path(&t, avatar_path).await.unwrap();
+    let maybe_sticker = &mut false;
+    let strict_limits = true;
+    blob.recode_to_size(&t, None, maybe_sticker, 1000, 3000, strict_limits)
+        .unwrap();
+    let new_file_size = file_size(&blob.to_abs_path()).await;
+    assert!(new_file_size <= 3000);
+    assert!(new_file_size > 2000);
+    // The new file should be smaller:
+    assert!(new_file_size < scaled_avatar_size);
+    // And the original file should not be touched:
+    assert_eq!(file_size(avatar_path).await, scaled_avatar_size);
+    tokio::task::block_in_place(move || {
+        let img = ImageReader::open(blob.to_abs_path())
+            .unwrap()
+            .with_guessed_format()
+            .unwrap()
+            .decode()
+            .unwrap();
+        assert!(img.width() > 130);
+        assert_eq!(img.width(), img.height());
+    });
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_in_blobdir() {
+    let t = TestContext::new().await;
+    let avatar_src = t.get_blobdir().join("avatar.png");
+    fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES)
+        .await
+        .unwrap();
+
+    check_image_size(&avatar_src, 900, 900);
+
+    t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+        .await
+        .unwrap();
+    let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
+    assert!(
+        avatar_cfg.ends_with("9e7f409ac5c92b942cc4f31cee2770a.png"),
+        "Avatar file name {avatar_cfg} should end with its hash"
+    );
+
+    check_image_size(
+        avatar_cfg,
+        constants::BALANCED_AVATAR_SIZE,
+        constants::BALANCED_AVATAR_SIZE,
+    );
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_selfavatar_copy_without_recode() {
+    let t = TestContext::new().await;
+    let avatar_src = t.dir.path().join("avatar.png");
+    let avatar_bytes = include_bytes!("../../test-data/image/avatar64x64.png");
+    fs::write(&avatar_src, avatar_bytes).await.unwrap();
+    let avatar_blob = t.get_blobdir().join("e9b6c7a78aa2e4f415644f55a553e73.png");
+    assert!(!avatar_blob.exists());
+    t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+        .await
+        .unwrap();
+    assert!(avatar_blob.exists());
+    assert_eq!(
+        fs::metadata(&avatar_blob).await.unwrap().len(),
+        avatar_bytes.len() as u64
+    );
+    let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
+    assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_recode_image_1() {
+    let bytes = include_bytes!("../../test-data/image/avatar1000x1000.jpg");
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "0",
+        bytes,
+        extension: "jpg",
+        has_exif: true,
+        original_width: 1000,
+        original_height: 1000,
+        compressed_width: 1000,
+        compressed_height: 1000,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "1",
+        bytes,
+        extension: "jpg",
+        has_exif: true,
+        original_width: 1000,
+        original_height: 1000,
+        compressed_width: 1000,
+        compressed_height: 1000,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_recode_image_2() {
+    // The "-rotated" files are rotated by 270 degrees using the Exif metadata
+    let bytes = include_bytes!("../../test-data/image/rectangle2000x1800-rotated.jpg");
+    let img_rotated = SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "0",
+        bytes,
+        extension: "jpg",
+        has_exif: true,
+        original_width: 2000,
+        original_height: 1800,
+        orientation: 270,
+        compressed_width: 1800,
+        compressed_height: 2000,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+    assert_correct_rotation(&img_rotated);
+
+    let mut buf = Cursor::new(vec![]);
+    img_rotated.write_to(&mut buf, ImageFormat::Jpeg).unwrap();
+    let bytes = buf.into_inner();
+
+    let img_rotated = SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "1",
+        bytes: &bytes,
+        extension: "jpg",
+        original_width: 1800,
+        original_height: 2000,
+        compressed_width: 1800,
+        compressed_height: 2000,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+    assert_correct_rotation(&img_rotated);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_recode_image_balanced_png() {
+    let bytes = include_bytes!("../../test-data/image/screenshot.png");
+
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "0",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: 1920,
+        compressed_height: 1080,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "1",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: constants::WORSE_IMAGE_SIZE,
+        compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::File,
+        media_quality_config: "1",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: 1920,
+        compressed_height: 1080,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::File,
+        media_quality_config: "1",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: 1920,
+        compressed_height: 1080,
+        set_draft: true,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+
+    // This will be sent as Image, see [`BlobObject::maybe_sticker`] for explanation.
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Sticker,
+        media_quality_config: "0",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: 1920,
+        compressed_height: 1080,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+}
+
+/// Tests that RGBA PNG can be recoded into JPEG
+/// by dropping alpha channel.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_recode_image_rgba_png_to_jpeg() {
+    let bytes = include_bytes!("../../test-data/image/screenshot-rgba.png");
+
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "1",
+        bytes,
+        extension: "png",
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: constants::WORSE_IMAGE_SIZE,
+        compressed_height: constants::WORSE_IMAGE_SIZE * 1080 / 1920,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_recode_image_huge_jpg() {
+    let bytes = include_bytes!("../../test-data/image/screenshot.jpg");
+    SendImageCheckMediaquality {
+        viewtype: Viewtype::Image,
+        media_quality_config: "0",
+        bytes,
+        extension: "jpg",
+        has_exif: true,
+        original_width: 1920,
+        original_height: 1080,
+        compressed_width: constants::BALANCED_IMAGE_SIZE,
+        compressed_height: constants::BALANCED_IMAGE_SIZE * 1080 / 1920,
+        ..Default::default()
+    }
+    .test()
+    .await
+    .unwrap();
+}
+
+fn assert_correct_rotation(img: &DynamicImage) {
+    // The test images are black in the bottom left corner after correctly applying
+    // the EXIF orientation
+
+    let [luma] = img.get_pixel(10, 10).to_luma().0;
+    assert_eq!(luma, 255);
+    let [luma] = img.get_pixel(img.width() - 10, 10).to_luma().0;
+    assert_eq!(luma, 255);
+    let [luma] = img
+        .get_pixel(img.width() - 10, img.height() - 10)
+        .to_luma()
+        .0;
+    assert_eq!(luma, 255);
+    let [luma] = img.get_pixel(10, img.height() - 10).to_luma().0;
+    assert_eq!(luma, 0);
+}
+
+#[derive(Default)]
+struct SendImageCheckMediaquality<'a> {
+    pub(crate) viewtype: Viewtype,
+    pub(crate) media_quality_config: &'a str,
+    pub(crate) bytes: &'a [u8],
+    pub(crate) extension: &'a str,
+    pub(crate) has_exif: bool,
+    pub(crate) original_width: u32,
+    pub(crate) original_height: u32,
+    pub(crate) orientation: i32,
+    pub(crate) compressed_width: u32,
+    pub(crate) compressed_height: u32,
+    pub(crate) set_draft: bool,
+}
+
+impl SendImageCheckMediaquality<'_> {
+    pub(crate) async fn test(self) -> anyhow::Result<DynamicImage> {
+        let viewtype = self.viewtype;
+        let media_quality_config = self.media_quality_config;
+        let bytes = self.bytes;
+        let extension = self.extension;
+        let has_exif = self.has_exif;
+        let original_width = self.original_width;
+        let original_height = self.original_height;
+        let orientation = self.orientation;
+        let compressed_width = self.compressed_width;
+        let compressed_height = self.compressed_height;
+        let set_draft = self.set_draft;
+
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        alice
+            .set_config(Config::MediaQuality, Some(media_quality_config))
+            .await?;
+        let file = alice.get_blobdir().join("file").with_extension(extension);
+        let file_name = format!("file.{extension}");
+
+        fs::write(&file, &bytes)
+            .await
+            .context("failed to write file")?;
+        check_image_size(&file, original_width, original_height);
+
+        let (_, exif) = image_metadata(&std::fs::File::open(&file)?)?;
+        if has_exif {
+            let exif = exif.unwrap();
+            assert_eq!(exif_orientation(&exif, &alice), orientation);
+        } else {
+            assert!(exif.is_none());
+        }
+
+        let mut msg = Message::new(viewtype);
+        msg.set_file_and_deduplicate(&alice, &file, Some(&file_name), None)?;
+        let chat = alice.create_chat(&bob).await;
+        if set_draft {
+            chat.id.set_draft(&alice, Some(&mut msg)).await.unwrap();
+            msg = chat.id.get_draft(&alice).await.unwrap().unwrap();
+            assert_eq!(msg.get_viewtype(), Viewtype::File);
+        }
+        let sent = alice.send_msg(chat.id, &mut msg).await;
+        let alice_msg = alice.get_last_msg().await;
+        assert_eq!(alice_msg.get_width() as u32, compressed_width);
+        assert_eq!(alice_msg.get_height() as u32, compressed_height);
+        let file_saved = alice
+            .get_blobdir()
+            .join("saved-".to_string() + &alice_msg.get_filename().unwrap());
+        alice_msg.save_file(&alice, &file_saved).await?;
+        check_image_size(file_saved, compressed_width, compressed_height);
+
+        let bob_msg = bob.recv_msg(&sent).await;
+        assert_eq!(bob_msg.get_viewtype(), Viewtype::Image);
+        assert_eq!(bob_msg.get_width() as u32, compressed_width);
+        assert_eq!(bob_msg.get_height() as u32, compressed_height);
+        let file_saved = bob
+            .get_blobdir()
+            .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+        bob_msg.save_file(&bob, &file_saved).await?;
+        if viewtype == Viewtype::File {
+            assert_eq!(file_saved.extension().unwrap(), extension);
+            let bytes1 = fs::read(&file_saved).await?;
+            assert_eq!(&bytes1, bytes);
+        }
+
+        let (_, exif) = image_metadata(&std::fs::File::open(&file_saved)?)?;
+        assert!(exif.is_none());
+
+        let img = check_image_size(file_saved, compressed_width, compressed_height);
+        Ok(img)
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_send_big_gif_as_image() -> Result<()> {
+    let bytes = include_bytes!("../../test-data/image/screenshot.gif");
+    let (width, height) = (1920u32, 1080u32);
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+    alice
+        .set_config(
+            Config::MediaQuality,
+            Some(&(MediaQuality::Worse as i32).to_string()),
+        )
+        .await?;
+    let file = alice.get_blobdir().join("file").with_extension("gif");
+    fs::write(&file, &bytes)
+        .await
+        .context("failed to write file")?;
+    let mut msg = Message::new(Viewtype::Image);
+    msg.set_file_and_deduplicate(&alice, &file, Some("file.gif"), None)?;
+    let chat = alice.create_chat(&bob).await;
+    let sent = alice.send_msg(chat.id, &mut msg).await;
+    let bob_msg = bob.recv_msg(&sent).await;
+    // DC must detect the image as GIF and send it w/o reencoding.
+    assert_eq!(bob_msg.get_viewtype(), Viewtype::Gif);
+    assert_eq!(bob_msg.get_width() as u32, width);
+    assert_eq!(bob_msg.get_height() as u32, height);
+    let file_saved = bob
+        .get_blobdir()
+        .join("saved-".to_string() + &bob_msg.get_filename().unwrap());
+    bob_msg.save_file(&bob, &file_saved).await?;
+    let (file_size, _) = image_metadata(&std::fs::File::open(&file_saved)?)?;
+    assert_eq!(file_size, bytes.len() as u64);
+    check_image_size(file_saved, width, height);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_send_gif_as_sticker() -> Result<()> {
+    let bytes = include_bytes!("../../test-data/image/image100x50.gif");
+    let alice = &TestContext::new_alice().await;
+    let file = alice.get_blobdir().join("file").with_extension("gif");
+    fs::write(&file, &bytes)
+        .await
+        .context("failed to write file")?;
+    let mut msg = Message::new(Viewtype::Sticker);
+    msg.set_file_and_deduplicate(alice, &file, None, None)?;
+    let chat = alice.get_self_chat().await;
+    let sent = alice.send_msg(chat.id, &mut msg).await;
+    let msg = Message::load_from_db(alice, sent.sender_msg_id).await?;
+    // Message::force_sticker() wasn't used, still Viewtype::Sticker is preserved because of the
+    // extension.
+    assert_eq!(msg.get_viewtype(), Viewtype::Sticker);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_and_deduplicate() -> Result<()> {
+    let t = TestContext::new().await;
+
+    let path = t.get_blobdir().join("anyfile.dat");
+    fs::write(&path, b"bla").await?;
+    let blob = BlobObject::create_and_deduplicate(&t, &path, &path)?;
+    assert_eq!(blob.name, "$BLOBDIR/ce940175885d7b78f7b7e9f1396611f.dat");
+    assert_eq!(path.exists(), false);
+
+    assert_eq!(fs::read(&blob.to_abs_path()).await?, b"bla");
+
+    fs::write(&path, b"bla").await?;
+    let blob2 = BlobObject::create_and_deduplicate(&t, &path, &path)?;
+    assert_eq!(blob2.name, blob.name);
+
+    let path_outside_blobdir = t.dir.path().join("anyfile.dat");
+    fs::write(&path_outside_blobdir, b"bla").await?;
+    let blob3 =
+        BlobObject::create_and_deduplicate(&t, &path_outside_blobdir, &path_outside_blobdir)?;
+    assert!(path_outside_blobdir.exists());
+    assert_eq!(blob3.name, blob.name);
+
+    fs::write(&path, b"blabla").await?;
+    let blob4 = BlobObject::create_and_deduplicate(&t, &path, &path)?;
+    assert_ne!(blob4.name, blob.name);
+
+    fs::remove_dir_all(t.get_blobdir()).await?;
+    let blob5 =
+        BlobObject::create_and_deduplicate(&t, &path_outside_blobdir, &path_outside_blobdir)?;
+    assert_eq!(blob5.name, blob.name);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_and_deduplicate_from_bytes() -> Result<()> {
+    let t = TestContext::new().await;
+
+    fs::remove_dir(t.get_blobdir()).await?;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(&t, b"bla", "file")?;
+    assert_eq!(blob.name, "$BLOBDIR/ce940175885d7b78f7b7e9f1396611f");
+
+    assert_eq!(fs::read(&blob.to_abs_path()).await?, b"bla");
+    let modified1 = blob.to_abs_path().metadata()?.modified()?;
+
+    // Test that the modification time of the file is updated when a new file is created
+    // so that it's not deleted during housekeeping.
+    // We can't use SystemTime::shift() here because file creation uses the actual OS time,
+    // which we can't mock from our code.
+    tokio::time::sleep(Duration::from_millis(1100)).await;
+
+    let blob2 = BlobObject::create_and_deduplicate_from_bytes(&t, b"bla", "file")?;
+    assert_eq!(blob2.name, blob.name);
+
+    let modified2 = blob.to_abs_path().metadata()?.modified()?;
+    assert_ne!(modified1, modified2);
+    sql::housekeeping(&t).await?;
+    assert!(blob2.to_abs_path().exists());
+
+    // If we do shift the time by more than 1h, the blob file will be deleted during housekeeping:
+    SystemTime::shift(Duration::from_secs(65 * 60));
+    sql::housekeeping(&t).await?;
+    assert_eq!(blob2.to_abs_path().exists(), false);
+
+    let blob3 = BlobObject::create_and_deduplicate_from_bytes(&t, b"blabla", "file")?;
+    assert_ne!(blob3.name, blob.name);
+
+    {
+        // If something goes wrong and the blob file is overwritten,
+        // the correct content should be restored:
+        fs::write(blob3.to_abs_path(), b"bloblo").await?;
+
+        let blob4 = BlobObject::create_and_deduplicate_from_bytes(&t, b"blabla", "file")?;
+        let blob4_content = fs::read(blob4.to_abs_path()).await?;
+        assert_eq!(blob4_content, b"blabla");
+    }
+
+    Ok(())
+}

src/chatlist.rs

@@ -144,7 +144,7 @@ impl Chatlist {
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked!=1
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2 AND add_timestamp >= remove_timestamp)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                 (MessageState::OutDraft, query_contact_id, ChatVisibility::Pinned),
@@ -261,7 +261,7 @@ impl Chatlist {
                      WHERE c.id>9 AND c.id!=?
                        AND c.blocked=0
                        AND NOT c.archived=?
-                       AND (c.type!=? OR c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?))
+                       AND (c.type!=? OR c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=? AND add_timestamp >= remove_timestamp))
                      GROUP BY c.id
                      ORDER BY c.id=? DESC, c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
                     (
@@ -394,25 +394,32 @@ impl Chatlist {
             &chat_loaded
         };
 
-        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
-            let lastmsg = Message::load_from_db(context, lastmsg_id)
+        let lastmsg = if let Some(lastmsg_id) = lastmsg_id {
+            // Message may be deleted by the time we try to load it,
+            // so use `load_from_db_optional` instead of `load_from_db`.
+            Message::load_from_db_optional(context, lastmsg_id)
                 .await
-                .context("loading message failed")?;
+                .context("Loading message failed")?
+        } else {
+            None
+        };
+
+        let lastcontact = if let Some(lastmsg) = &lastmsg {
             if lastmsg.from_id == ContactId::SELF {
-                (Some(lastmsg), None)
+                None
             } else {
                 match chat.typ {
                     Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                         let lastcontact = Contact::get_by_id(context, lastmsg.from_id)
                             .await
                             .context("loading contact failed")?;
-                        (Some(lastmsg), Some(lastcontact))
+                        Some(lastcontact)
                     }
-                    Chattype::Single => (Some(lastmsg), None),
+                    Chattype::Single => None,
                 }
             }
         } else {
-            (None, None)
+            None
         };
 
         if chat.id.is_archived_link() {
@@ -543,7 +550,7 @@ mod tests {
         let chats = Chatlist::try_load(&t, 0, Some("is:unread"), None)
             .await
             .unwrap();
-        assert!(chats.len() == 1);
+        assert_eq!(chats.len(), 1);
 
         let chats = Chatlist::try_load(&t, DC_GCL_ARCHIVED_ONLY, None, None)
             .await
@@ -569,7 +576,7 @@ mod tests {
             .unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert!(chats.len() == 3);
+        assert_eq!(chats.len(), 3);
         assert!(!Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
             .await
             .unwrap()
@@ -578,7 +585,7 @@ mod tests {
         let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
-        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
+        assert_eq!(chats.len(), 2); // device chat cannot be written and is skipped on forwarding
         assert!(Chat::load_from_db(&t, chats.get_chat_id(0).unwrap())
             .await
             .unwrap()
@@ -590,7 +597,7 @@ mod tests {
         let chats = Chatlist::try_load(&t, DC_GCL_FOR_FORWARDING, None, None)
             .await
             .unwrap();
-        assert!(chats.len() == 1);
+        assert_eq!(chats.len(), 1);
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -761,6 +768,25 @@ mod tests {
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 
+    /// Tests that summary does not fail to load
+    /// if the draft was deleted after loading the chatlist.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_summary_deleted_draft() {
+        let t = TestContext::new().await;
+
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+            .await
+            .unwrap();
+        let mut msg = Message::new_text("Foobar".to_string());
+        chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
+
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        chat_id.set_draft(&t, None).await.unwrap();
+
+        let summary_res = chats.get_summary(&t, 0, None).await;
+        assert!(summary_res.is_ok());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_broken() {
         let t = TestContext::new_bob().await;

src/config.rs

@@ -80,7 +80,7 @@ pub enum Config {
     /// SMTP server security (e.g. TLS, STARTTLS).
     SendSecurity,
 
-    /// Deprecated option for backwards compatibilty.
+    /// Deprecated option for backwards compatibility.
     ///
     /// Certificate checks for SMTP are actually controlled by `imap_certificate_checks` config.
     SmtpCertificateChecks,
@@ -143,7 +143,10 @@ pub enum Config {
     /// Send BCC copy to self.
     ///
     /// Should be enabled for multidevice setups.
-    #[strum(props(default = "1"))]
+    /// Default is 0 for chatmail accounts, 1 otherwise.
+    ///
+    /// This is automatically enabled when importing/exporting a backup,
+    /// setting up a second device, or receiving a sync message.
     BccSelf,
 
     /// True if encryption is preferred according to Autocrypt standard.
@@ -202,7 +205,7 @@ pub enum Config {
     /// Value 1 is treated as "delete at once": messages are deleted
     /// immediately, without moving to DeltaChat folder.
     ///
-    /// Default is 1 for chatmail accounts before a backup export, 0 otherwise.
+    /// Default is 1 for chatmail accounts without `BccSelf`, 0 otherwise.
     DeleteServerAfter,
 
     /// Timer in seconds after which the message is deleted from the
@@ -384,6 +387,11 @@ pub enum Config {
 
     /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set
     /// and `Bot` unset.
+    ///
+    /// On real devices, this is usually always enabled and `BccSelf` is the only setting
+    /// that controls whether sync messages are sent.
+    ///
+    /// In tests, this is usually disabled.
     #[strum(props(default = "1"))]
     SyncMsgs,
 
@@ -441,6 +449,19 @@ pub enum Config {
     /// Enable webxdc realtime features.
     #[strum(props(default = "1"))]
     WebxdcRealtimeEnabled,
+
+    /// Last device token stored on the chatmail server.
+    ///
+    /// If it has not changed, we do not store
+    /// the device token again.
+    DeviceToken,
+
+    /// Device token encrypted with OpenPGP.
+    ///
+    /// We store encrypted token next to `device_token`
+    /// to avoid encrypting it differently and
+    /// storing the same token multiple times on the server.
+    EncryptedDeviceToken,
 }
 
 impl Config {
@@ -513,11 +534,19 @@ impl Context {
 
         // Default values
         let val = match key {
-            Config::ConfiguredInboxFolder => Some("INBOX"),
-            Config::DeleteServerAfter => match Box::pin(self.is_chatmail()).await? {
-                false => Some("0"),
-                true => Some("1"),
+            Config::BccSelf => match Box::pin(self.is_chatmail()).await? {
+                false => Some("1"),
+                true => Some("0"),
             },
+            Config::ConfiguredInboxFolder => Some("INBOX"),
+            Config::DeleteServerAfter => {
+                match !Box::pin(self.get_config_bool(Config::BccSelf)).await?
+                    && Box::pin(self.is_chatmail()).await?
+                {
+                    true => Some("1"),
+                    false => Some("0"),
+                }
+            }
             _ => key.get_str("default"),
         };
         Ok(val.map(|s| s.to_string()))
@@ -664,7 +693,7 @@ impl Context {
         let value = match key {
             Config::Selfavatar if value.is_empty() => None,
             Config::Selfavatar => {
-                config_value = BlobObject::store_from_base64(self, value, "avatar").await?;
+                config_value = BlobObject::store_from_base64(self, value)?;
                 Some(config_value.as_str())
             }
             _ => Some(value),
@@ -742,7 +771,8 @@ impl Context {
                     .await?;
                 match value {
                     Some(path) => {
-                        let mut blob = BlobObject::new_from_path(self, path.as_ref()).await?;
+                        let path = get_abs_path(self, Path::new(path));
+                        let mut blob = BlobObject::create_and_deduplicate(self, &path, &path)?;
                         blob.recode_to_avatar_size(self).await?;
                         self.sql
                             .set_raw_config(key.as_ref(), Some(blob.as_name()))
@@ -791,6 +821,12 @@ impl Context {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
+        if matches!(
+            key,
+            Config::Displayname | Config::Selfavatar | Config::PrivateTag
+        ) {
+            self.emit_event(EventType::AccountsItemChanged);
+        }
         if key.is_synced() {
             self.emit_event(EventType::ConfigSynced { key });
         }
@@ -1093,6 +1129,30 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_delete_server_after_default() -> Result<()> {
+        let t = &TestContext::new_alice().await;
+
+        // Check that the settings are displayed correctly.
+        assert_eq!(t.get_config(Config::BccSelf).await?, Some("1".to_string()));
+        assert_eq!(
+            t.get_config(Config::DeleteServerAfter).await?,
+            Some("0".to_string())
+        );
+
+        // Leaving emails on the server even w/o `BccSelf` is a good default at least because other
+        // MUAs do so even if the server doesn't save sent messages to some sentbox (like Gmail
+        // does).
+        t.set_config_bool(Config::BccSelf, false).await?;
+        assert_eq!(
+            t.get_config(Config::DeleteServerAfter).await?,
+            Some("0".to_string())
+        );
+        Ok(())
+    }
+
+    const SAVED_MESSAGES_DEDUPLICATED_FILE: &str = "969142cb84015bc135767bc2370934a.png";
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;
@@ -1167,7 +1227,7 @@ mod tests {
         let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
         assert_eq!(
             self_chat_avatar_path,
-            alice0.get_blobdir().join("icon-saved-messages.png")
+            alice0.get_blobdir().join(SAVED_MESSAGES_DEDUPLICATED_FILE)
         );
         assert!(alice1
             .get_config(Config::Selfavatar)

src/configure.rs

@@ -36,10 +36,10 @@ use crate::message::Message;
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::time;
 use crate::{chat, e2ee, provider};
+use crate::{stock_str, EventType};
 use deltachat_contact_tools::addr_cmp;
 
 macro_rules! progress {
@@ -61,10 +61,7 @@ macro_rules! progress {
 impl Context {
     /// Checks if the context is already configured.
     pub async fn is_configured(&self) -> Result<bool> {
-        self.sql
-            .get_raw_config_bool("configured")
-            .await
-            .map_err(Into::into)
+        self.sql.get_raw_config_bool("configured").await
     }
 
     /// Configures this account with the currently set parameters.
@@ -452,8 +449,9 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
     imap.configure_folders(ctx, &mut imap_session, create_mvbox)
         .await?;
 
+    let create = true;
     imap_session
-        .select_with_uidvalidity(ctx, "INBOX")
+        .select_with_uidvalidity(ctx, "INBOX", create)
         .await
         .context("could not read INBOX status")?;
 
@@ -486,6 +484,7 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
     update_device_chats_handle.await??;
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
+    ctx.emit_event(EventType::AccountsItemChanged);
 
     Ok(configured_param)
 }
@@ -611,8 +610,6 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
     use crate::config::Config;
     use crate::login_param::EnteredServerLoginParam;

src/configure/auto_mozilla.rs

@@ -67,19 +67,15 @@ fn parse_server<B: BufRead>(
 
     let typ = server_event
         .attributes()
-        .find(|attr| {
-            attr.as_ref()
-                .map(|a| {
-                    String::from_utf8_lossy(a.key.as_ref())
-                        .trim()
-                        .to_lowercase()
-                        == "type"
-                })
-                .unwrap_or_default()
+        .find_map(|attr| {
+            attr.ok().filter(|a| {
+                String::from_utf8_lossy(a.key.as_ref())
+                    .trim()
+                    .eq_ignore_ascii_case("type")
+            })
         })
         .map(|typ| {
-            typ.unwrap()
-                .decode_and_unescape_value(reader.decoder())
+            typ.decode_and_unescape_value(reader.decoder())
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -272,8 +268,6 @@ pub(crate) async fn moz_autoconfigure(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/configure/auto_outlook.rs

@@ -215,8 +215,6 @@ pub(crate) async fn outlk_autodiscover(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/context.rs

@@ -10,9 +10,10 @@ use std::time::Duration;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
+use pgp::types::PublicKeyTrait;
 use pgp::SignedPublicKey;
 use ratelimit::Ratelimit;
-use tokio::sync::{Mutex, Notify, OnceCell, RwLock};
+use tokio::sync::{Mutex, Notify, RwLock};
 
 use crate::aheader::EncryptPreference;
 use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus};
@@ -275,7 +276,7 @@ pub struct InnerContext {
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
     /// `last_error` should be used to avoid races with the event thread.
-    pub(crate) last_error: std::sync::RwLock<String>,
+    pub(crate) last_error: parking_lot::RwLock<String>,
 
     /// If debug logging is enabled, this contains all necessary information
     ///
@@ -291,7 +292,7 @@ pub struct InnerContext {
     pub(crate) push_subscribed: AtomicBool,
 
     /// Iroh for realtime peer channels.
-    pub(crate) iroh: OnceCell<Iroh>,
+    pub(crate) iroh: Arc<RwLock<Option<Iroh>>>,
 }
 
 /// The state of ongoing process.
@@ -445,11 +446,11 @@ impl Context {
             metadata: RwLock::new(None),
             creation_time: tools::Time::now(),
             last_full_folder_scan: Mutex::new(None),
-            last_error: std::sync::RwLock::new("".to_string()),
+            last_error: parking_lot::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
             push_subscriber,
             push_subscribed: AtomicBool::new(false),
-            iroh: OnceCell::new(),
+            iroh: Arc::new(RwLock::new(None)),
         };
 
         let ctx = Context {
@@ -471,12 +472,33 @@ impl Context {
             // Allow at least 1 message every second + a burst of 3.
             *lock = Ratelimit::new(Duration::new(3, 0), 3.0);
         }
+
+        // The next line is mainly for iOS:
+        // iOS starts a separate process for receiving notifications and if the user concurrently
+        // starts the app, the UI process opens the database but waits with calling start_io()
+        // until the notifications process finishes.
+        // Now, some configs may have changed, so, we need to invalidate the cache.
+        self.sql.config_cache.write().await.clear();
+
         self.scheduler.start(self.clone()).await;
     }
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         self.scheduler.stop(self).await;
+        if let Some(iroh) = self.iroh.write().await.take() {
+            // Close all QUIC connections.
+
+            // Spawn into a separate task,
+            // because Iroh calls `wait_idle()` internally
+            // and it may take time, especially if the network
+            // has become unavailable.
+            tokio::spawn(async move {
+                // We do not log the error because we do not want the task
+                // to hold the reference to Context.
+                let _ = tokio::time::timeout(Duration::from_secs(60), iroh.close()).await;
+            });
+        }
     }
 
     /// Restarts the IO scheduler if it was running before
@@ -487,7 +509,7 @@ impl Context {
 
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        if let Some(iroh) = self.iroh.get() {
+        if let Some(ref iroh) = *self.iroh.read().await {
             iroh.network_change().await;
         }
         self.scheduler.maybe_network().await;
@@ -531,23 +553,7 @@ impl Context {
 
         if self.scheduler.is_running().await {
             self.scheduler.maybe_network().await;
-
-            // Wait until fetching is finished.
-            // Ideally we could wait for connectivity change events,
-            // but sleep loop is good enough.
-
-            // First 100 ms sleep in chunks of 10 ms.
-            for _ in 0..10 {
-                if self.all_work_done().await {
-                    break;
-                }
-                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
-            }
-
-            // If we are not finished in 100 ms, keep waking up every 100 ms.
-            while !self.all_work_done().await {
-                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-            }
+            self.wait_for_all_work_done().await;
         } else {
             // Pause the scheduler to ensure another connection does not start
             // while we are fetching on a dedicated connection.
@@ -637,14 +643,36 @@ impl Context {
     }
 
     /// Emits a MsgsChanged event with specified chat and message ids
+    ///
+    /// If IDs are unset, [`Self::emit_msgs_changed_without_ids`]
+    /// or [`Self::emit_msgs_changed_without_msg_id`] should be used
+    /// instead of this function.
     pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
+        debug_assert!(!chat_id.is_unset());
+        debug_assert!(!msg_id.is_unset());
+
         self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
         chatlist_events::emit_chatlist_changed(self);
         chatlist_events::emit_chatlist_item_changed(self, chat_id);
     }
 
+    /// Emits a MsgsChanged event with specified chat and without message id.
+    pub fn emit_msgs_changed_without_msg_id(&self, chat_id: ChatId) {
+        debug_assert!(!chat_id.is_unset());
+
+        self.emit_event(EventType::MsgsChanged {
+            chat_id,
+            msg_id: MsgId::new(0),
+        });
+        chatlist_events::emit_chatlist_changed(self);
+        chatlist_events::emit_chatlist_item_changed(self, chat_id);
+    }
+
     /// Emits an IncomingMsg event with specified chat and message ids
     pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
+        debug_assert!(!chat_id.is_unset());
+        debug_assert!(!msg_id.is_unset());
+
         self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
         chatlist_events::emit_chatlist_changed(self);
         chatlist_events::emit_chatlist_item_changed(self, chat_id);
@@ -781,7 +809,7 @@ impl Context {
             .count("SELECT COUNT(*) FROM acpeerstates;", ())
             .await?;
         let fingerprint_str = match load_self_public_key(self).await {
-            Ok(key) => key.fingerprint().hex(),
+            Ok(key) => key.dc_fingerprint().hex(),
             Err(err) => format!("<key failure: {err}>"),
         };
 
@@ -1177,7 +1205,7 @@ impl Context {
             EncryptPreference::Mutual,
             &public_key,
         );
-        let fingerprint = public_key.fingerprint();
+        let fingerprint = public_key.dc_fingerprint();
         peerstate.set_verified(public_key, fingerprint, "".to_string())?;
         peerstate.save_to_db(&self.sql).await?;
         chat_id
@@ -1452,615 +1480,4 @@ pub fn get_version_str() -> &'static str {
 }
 
 #[cfg(test)]
-mod tests {
-    use anyhow::Context as _;
-    use strum::IntoEnumIterator;
-    use tempfile::tempdir;
-
-    use super::*;
-    use crate::chat::{get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, MuteDuration};
-    use crate::chatlist::Chatlist;
-    use crate::constants::Chattype;
-    use crate::mimeparser::SystemMessage;
-    use crate::receive_imf::receive_imf;
-    use crate::test_utils::{get_chat_msg, TestContext};
-    use crate::tools::{create_outgoing_rfc724_mid, SystemTime};
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_wrong_db() -> Result<()> {
-        let tmp = tempfile::tempdir()?;
-        let dbfile = tmp.path().join("db.sqlite");
-        tokio::fs::write(&dbfile, b"123").await?;
-        let res = Context::new(&dbfile, 1, Events::new(), StockStrings::new()).await?;
-
-        // Broken database is indistinguishable from encrypted one.
-        assert_eq!(res.is_open().await, false);
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_fresh_msgs() {
-        let t = TestContext::new().await;
-        let fresh = t.get_fresh_msgs().await.unwrap();
-        assert!(fresh.is_empty())
-    }
-
-    async fn receive_msg(t: &TestContext, chat: &Chat) {
-        let members = get_chat_contacts(t, chat.id).await.unwrap();
-        let contact = Contact::get_by_id(t, *members.first().unwrap())
-            .await
-            .unwrap();
-        let msg = format!(
-            "From: {}\n\
-             To: alice@example.org\n\
-             Message-ID: <{}>\n\
-             Chat-Version: 1.0\n\
-             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-             \n\
-             hello\n",
-            contact.get_addr(),
-            create_outgoing_rfc724_mid()
-        );
-        println!("{msg}");
-        receive_imf(t, msg.as_bytes(), false).await.unwrap();
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_fresh_msgs_and_muted_chats() {
-        // receive various mails in 3 chats
-        let t = TestContext::new_alice().await;
-        let bob = t.create_chat_with_contact("", "bob@g.it").await;
-        let claire = t.create_chat_with_contact("", "claire@g.it").await;
-        let dave = t.create_chat_with_contact("", "dave@g.it").await;
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        receive_msg(&t, &bob).await;
-        assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1);
-        assert_eq!(bob.id.get_fresh_msg_cnt(&t).await.unwrap(), 1);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        receive_msg(&t, &claire).await;
-        receive_msg(&t, &claire).await;
-        assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 2);
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
-
-        receive_msg(&t, &dave).await;
-        receive_msg(&t, &dave).await;
-        receive_msg(&t, &dave).await;
-        assert_eq!(get_chat_msgs(&t, dave.id).await.unwrap().len(), 3);
-        assert_eq!(dave.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
-
-        // mute one of the chats
-        set_muted(&t, claire.id, MuteDuration::Forever)
-            .await
-            .unwrap();
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
-
-        // receive more messages
-        receive_msg(&t, &bob).await;
-        receive_msg(&t, &claire).await;
-        receive_msg(&t, &dave).await;
-        assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 3);
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
-
-        // unmute claire again
-        set_muted(&t, claire.id, MuteDuration::NotMuted)
-            .await
-            .unwrap();
-        assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_fresh_msgs_and_muted_until() {
-        let t = TestContext::new_alice().await;
-        let bob = t.create_chat_with_contact("", "bob@g.it").await;
-        receive_msg(&t, &bob).await;
-        assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1);
-
-        // chat is unmuted by default, here and in the following assert(),
-        // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
-        // have the same view to the database.
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        // test get_fresh_msgs() with mute_until in the future
-        set_muted(
-            &t,
-            bob.id,
-            MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
-        )
-        .await
-        .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        // to test get_fresh_msgs() with mute_until in the past,
-        // we need to modify the database directly
-        t.sql
-            .execute(
-                "UPDATE chats SET muted_until=? WHERE id=?;",
-                (time() - 3600, bob.id),
-            )
-            .await
-            .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        // test get_fresh_msgs() with "forever" mute_until
-        set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-
-        // to test get_fresh_msgs() with invalid mute_until (everything < -1),
-        // that results in "muted forever" by definition.
-        t.sql
-            .execute("UPDATE chats SET muted_until=-2 WHERE id=?;", (bob.id,))
-            .await
-            .unwrap();
-        let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
-        assert!(!bob.is_muted());
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_muted_context() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
-        t.set_config(Config::IsMuted, Some("1")).await?;
-        let chat = t.create_chat_with_contact("", "bob@g.it").await;
-        receive_msg(&t, &chat).await;
-
-        // muted contexts should still show dimmed badge counters eg. in the sidebars,
-        // (same as muted chats show dimmed badge counters in the chatlist)
-        // therefore the fresh messages count should not be affected.
-        assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_blobdir_exists() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dbfile = tmp.path().join("db.sqlite");
-        Context::new(&dbfile, 1, Events::new(), StockStrings::new())
-            .await
-            .unwrap();
-        let blobdir = tmp.path().join("db.sqlite-blobs");
-        assert!(blobdir.is_dir());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_wrong_blogdir() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dbfile = tmp.path().join("db.sqlite");
-        let blobdir = tmp.path().join("db.sqlite-blobs");
-        tokio::fs::write(&blobdir, b"123").await.unwrap();
-        let res = Context::new(&dbfile, 1, Events::new(), StockStrings::new()).await;
-        assert!(res.is_err());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_sqlite_parent_not_exists() {
-        let tmp = tempfile::tempdir().unwrap();
-        let subdir = tmp.path().join("subdir");
-        let dbfile = subdir.join("db.sqlite");
-        let dbfile2 = dbfile.clone();
-        Context::new(&dbfile, 1, Events::new(), StockStrings::new())
-            .await
-            .unwrap();
-        assert!(subdir.is_dir());
-        assert!(dbfile2.is_file());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_with_empty_blobdir() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dbfile = tmp.path().join("db.sqlite");
-        let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(
-            dbfile,
-            blobdir,
-            1,
-            Events::new(),
-            StockStrings::new(),
-            Default::default(),
-        );
-        assert!(res.is_err());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_with_blobdir_not_exists() {
-        let tmp = tempfile::tempdir().unwrap();
-        let dbfile = tmp.path().join("db.sqlite");
-        let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(
-            dbfile,
-            blobdir,
-            1,
-            Events::new(),
-            StockStrings::new(),
-            Default::default(),
-        );
-        assert!(res.is_err());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn no_crashes_on_context_deref() {
-        let t = TestContext::new().await;
-        std::mem::drop(t);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_info() {
-        let t = TestContext::new().await;
-
-        let info = t.get_info().await.unwrap();
-        assert!(info.contains_key("database_dir"));
-    }
-
-    #[test]
-    fn test_get_info_no_context() {
-        let info = get_info();
-        assert!(info.contains_key("deltachat_core_version"));
-        assert!(!info.contains_key("database_dir"));
-        assert_eq!(info.get("level").unwrap(), "awesome");
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_info_completeness() {
-        // For easier debugging,
-        // get_info() shall return all important information configurable by the Config-values.
-        //
-        // There are exceptions for Config-values considered to be unimportant,
-        // too sensitive or summarized in another item.
-        let skip_from_get_info = vec![
-            "addr",
-            "displayname",
-            "imap_certificate_checks",
-            "mail_server",
-            "mail_user",
-            "mail_pw",
-            "mail_port",
-            "mail_security",
-            "notify_about_wrong_pw",
-            "self_reporting_id",
-            "selfstatus",
-            "send_server",
-            "send_user",
-            "send_pw",
-            "send_port",
-            "send_security",
-            "server_flags",
-            "skip_start_messages",
-            "smtp_certificate_checks",
-            "proxy_url",      // May contain passwords, don't leak it to the logs.
-            "socks5_enabled", // SOCKS5 options are deprecated.
-            "socks5_host",
-            "socks5_port",
-            "socks5_user",
-            "socks5_password",
-            "key_id",
-            "webxdc_integration",
-        ];
-        let t = TestContext::new().await;
-        let info = t.get_info().await.unwrap();
-        for key in Config::iter() {
-            let key: String = key.to_string();
-            if !skip_from_get_info.contains(&&*key)
-                && !key.starts_with("configured")
-                && !key.starts_with("sys.")
-            {
-                assert!(
-                    info.contains_key(&*key),
-                    "'{key}' missing in get_info() output"
-                );
-            }
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_search_msgs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
-        let chat = alice
-            .create_chat_with_contact("Bob", "bob@example.org")
-            .await;
-
-        // Global search finds nothing.
-        let res = alice.search_msgs(None, "foo").await?;
-        assert!(res.is_empty());
-
-        // Search in chat with Bob finds nothing.
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert!(res.is_empty());
-
-        // Add messages to chat with Bob.
-        let mut msg1 = Message::new_text("foobar".to_string());
-        send_msg(&alice, chat.id, &mut msg1).await?;
-
-        let mut msg2 = Message::new_text("barbaz".to_string());
-        send_msg(&alice, chat.id, &mut msg2).await?;
-
-        alice.send_text(chat.id, "Δ-Chat").await;
-
-        // Global search with a part of text finds the message.
-        let res = alice.search_msgs(None, "ob").await?;
-        assert_eq!(res.len(), 1);
-
-        // Global search for "bar" matches both "foobar" and "barbaz".
-        let res = alice.search_msgs(None, "bar").await?;
-        assert_eq!(res.len(), 2);
-
-        // Message added later is returned first.
-        assert_eq!(res.first(), Some(&msg2.id));
-        assert_eq!(res.get(1), Some(&msg1.id));
-
-        // Search is case-insensitive.
-        for chat_id in [None, Some(chat.id)] {
-            let res = alice.search_msgs(chat_id, "δ-chat").await?;
-            assert_eq!(res.len(), 1);
-        }
-
-        // Global search with longer text does not find any message.
-        let res = alice.search_msgs(None, "foobarbaz").await?;
-        assert!(res.is_empty());
-
-        // Search for random string finds nothing.
-        let res = alice.search_msgs(None, "abc").await?;
-        assert!(res.is_empty());
-
-        // Search in chat with Bob finds the message.
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert_eq!(res.len(), 1);
-
-        // Search in Saved Messages does not find the message.
-        let res = alice.search_msgs(Some(self_talk), "foo").await?;
-        assert!(res.is_empty());
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_search_unaccepted_requests() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        receive_imf(
-            &t,
-            b"From: BobBar <bob@example.org>\n\
-                 To: alice@example.org\n\
-                 Subject: foo\n\
-                 Message-ID: <msg1234@example.org>\n\
-                 Chat-Version: 1.0\n\
-                 Date: Tue, 25 Oct 2022 13:37:00 +0000\n\
-                 \n\
-                 hello bob, foobar test!\n",
-            false,
-        )
-        .await?;
-        let chat_id = t.get_last_msg().await.get_chat_id();
-        let chat = Chat::load_from_db(&t, chat_id).await?;
-        assert_eq!(chat.get_type(), Chattype::Single);
-        assert!(chat.is_contact_request());
-
-        assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 1);
-        assert_eq!(
-            Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
-            1
-        );
-        assert_eq!(t.search_msgs(None, "foobar").await?.len(), 1);
-        assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 1);
-
-        chat_id.block(&t).await?;
-
-        assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 0);
-        assert_eq!(
-            Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
-            0
-        );
-        assert_eq!(t.search_msgs(None, "foobar").await?.len(), 0);
-        assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 0);
-
-        let contact_ids = get_chat_contacts(&t, chat_id).await?;
-        Contact::unblock(&t, *contact_ids.first().unwrap()).await?;
-
-        assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 1);
-        assert_eq!(
-            Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
-            1
-        );
-        assert_eq!(t.search_msgs(None, "foobar").await?.len(), 1);
-        assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 1);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_limit_search_msgs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let chat = alice
-            .create_chat_with_contact("Bob", "bob@example.org")
-            .await;
-
-        // Add 999 messages
-        let mut msg = Message::new_text("foobar".to_string());
-        for _ in 0..999 {
-            send_msg(&alice, chat.id, &mut msg).await?;
-        }
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 999);
-
-        // Add one more message, no limit yet
-        send_msg(&alice, chat.id, &mut msg).await?;
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 1000);
-
-        // Add one more message, that one is truncated then
-        send_msg(&alice, chat.id, &mut msg).await?;
-        let res = alice.search_msgs(None, "foo").await?;
-        assert_eq!(res.len(), 1000);
-
-        // In-chat should not be not limited
-        let res = alice.search_msgs(Some(chat.id), "foo").await?;
-        assert_eq!(res.len(), 1001);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_check_passphrase() -> Result<()> {
-        let dir = tempdir()?;
-        let dbfile = dir.path().join("db.sqlite");
-
-        let context = ContextBuilder::new(dbfile.clone())
-            .with_id(1)
-            .build()
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.open("foo".to_string()).await?, true);
-        assert_eq!(context.is_open().await, true);
-        drop(context);
-
-        let context = ContextBuilder::new(dbfile)
-            .with_id(2)
-            .build()
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.is_open().await, false);
-        assert_eq!(context.check_passphrase("bar".to_string()).await?, false);
-        assert_eq!(context.open("false".to_string()).await?, false);
-        assert_eq!(context.open("foo".to_string()).await?, true);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_context_change_passphrase() -> Result<()> {
-        let dir = tempdir()?;
-        let dbfile = dir.path().join("db.sqlite");
-
-        let context = ContextBuilder::new(dbfile)
-            .with_id(1)
-            .build()
-            .await
-            .context("failed to create context")?;
-        assert_eq!(context.open("foo".to_string()).await?, true);
-        assert_eq!(context.is_open().await, true);
-
-        context
-            .set_config(Config::Addr, Some("alice@example.org"))
-            .await?;
-
-        context
-            .change_passphrase("bar".to_string())
-            .await
-            .context("Failed to change passphrase")?;
-
-        assert_eq!(
-            context.get_config(Config::Addr).await?.unwrap(),
-            "alice@example.org"
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ongoing() -> Result<()> {
-        let context = TestContext::new().await;
-
-        // No ongoing process allocated.
-        assert!(context.shall_stop_ongoing().await);
-
-        let receiver = context.alloc_ongoing().await?;
-
-        // Cannot allocate another ongoing process while the first one is running.
-        assert!(context.alloc_ongoing().await.is_err());
-
-        // Stop signal is not sent yet.
-        assert!(receiver.try_recv().is_err());
-
-        assert!(!context.shall_stop_ongoing().await);
-
-        // Send the stop signal.
-        context.stop_ongoing().await;
-
-        // Receive stop signal.
-        receiver.recv().await?;
-
-        assert!(context.shall_stop_ongoing().await);
-
-        // Ongoing process is still running even though stop signal was received,
-        // so another one cannot be allocated.
-        assert!(context.alloc_ongoing().await.is_err());
-
-        context.free_ongoing().await;
-
-        // No ongoing process allocated, should have been stopped already.
-        assert!(context.shall_stop_ongoing().await);
-
-        // Another ongoing process can be allocated now.
-        let _receiver = context.alloc_ongoing().await?;
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_next_msgs() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let alice_chat = alice.create_chat(&bob).await;
-
-        assert!(alice.get_next_msgs().await?.is_empty());
-        assert!(bob.get_next_msgs().await?.is_empty());
-
-        let sent_msg = alice.send_text(alice_chat.id, "Hi Bob").await;
-        let received_msg = bob.recv_msg(&sent_msg).await;
-
-        let bob_next_msg_ids = bob.get_next_msgs().await?;
-        assert_eq!(bob_next_msg_ids.len(), 1);
-        assert_eq!(bob_next_msg_ids.first(), Some(&received_msg.id));
-
-        bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
-            .await?;
-        assert!(bob.get_next_msgs().await?.is_empty());
-
-        // Next messages include self-sent messages.
-        let alice_next_msg_ids = alice.get_next_msgs().await?;
-        assert_eq!(alice_next_msg_ids.len(), 1);
-        assert_eq!(alice_next_msg_ids.first(), Some(&sent_msg.sender_msg_id));
-
-        alice
-            .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())
-            .await?;
-        assert!(alice.get_next_msgs().await?.is_empty());
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_draft_self_report() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        let chat_id = alice.draft_self_report().await?;
-        let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
-        assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
-
-        let chat = Chat::load_from_db(&alice, chat_id).await?;
-        assert!(chat.is_protected());
-
-        let mut draft = chat_id.get_draft(&alice).await?.unwrap();
-        assert!(draft.text.starts_with("core_version"));
-
-        // Test that sending into the protected chat works:
-        let _sent = alice.send_msg(chat_id, &mut draft).await;
-
-        Ok(())
-    }
-}
+mod context_tests;

src/context/context_tests.rs

@@ -0,0 +1,649 @@
+use anyhow::Context as _;
+use strum::IntoEnumIterator;
+use tempfile::tempdir;
+
+use super::*;
+use crate::chat::{get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, MuteDuration};
+use crate::chatlist::Chatlist;
+use crate::constants::Chattype;
+use crate::mimeparser::SystemMessage;
+use crate::receive_imf::receive_imf;
+use crate::test_utils::{get_chat_msg, TestContext};
+use crate::tools::{create_outgoing_rfc724_mid, SystemTime};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_wrong_db() -> Result<()> {
+    let tmp = tempfile::tempdir()?;
+    let dbfile = tmp.path().join("db.sqlite");
+    tokio::fs::write(&dbfile, b"123").await?;
+    let res = Context::new(&dbfile, 1, Events::new(), StockStrings::new()).await?;
+
+    // Broken database is indistinguishable from encrypted one.
+    assert_eq!(res.is_open().await, false);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_fresh_msgs() {
+    let t = TestContext::new().await;
+    let fresh = t.get_fresh_msgs().await.unwrap();
+    assert!(fresh.is_empty())
+}
+
+async fn receive_msg(t: &TestContext, chat: &Chat) {
+    let members = get_chat_contacts(t, chat.id).await.unwrap();
+    let contact = Contact::get_by_id(t, *members.first().unwrap())
+        .await
+        .unwrap();
+    let msg = format!(
+        "From: {}\n\
+             To: alice@example.org\n\
+             Message-ID: <{}>\n\
+             Chat-Version: 1.0\n\
+             Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
+             \n\
+             hello\n",
+        contact.get_addr(),
+        create_outgoing_rfc724_mid()
+    );
+    println!("{msg}");
+    receive_imf(t, msg.as_bytes(), false).await.unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_fresh_msgs_and_muted_chats() {
+    // receive various mails in 3 chats
+    let t = TestContext::new_alice().await;
+    let bob = t.create_chat_with_contact("", "bob@g.it").await;
+    let claire = t.create_chat_with_contact("", "claire@g.it").await;
+    let dave = t.create_chat_with_contact("", "dave@g.it").await;
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+    receive_msg(&t, &bob).await;
+    assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1);
+    assert_eq!(bob.id.get_fresh_msg_cnt(&t).await.unwrap(), 1);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+    receive_msg(&t, &claire).await;
+    receive_msg(&t, &claire).await;
+    assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 2);
+    assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 3);
+
+    receive_msg(&t, &dave).await;
+    receive_msg(&t, &dave).await;
+    receive_msg(&t, &dave).await;
+    assert_eq!(get_chat_msgs(&t, dave.id).await.unwrap().len(), 3);
+    assert_eq!(dave.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6);
+
+    // mute one of the chats
+    set_muted(&t, claire.id, MuteDuration::Forever)
+        .await
+        .unwrap();
+    assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 2);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 4); // muted claires messages are no longer counted
+
+    // receive more messages
+    receive_msg(&t, &bob).await;
+    receive_msg(&t, &claire).await;
+    receive_msg(&t, &dave).await;
+    assert_eq!(get_chat_msgs(&t, claire.id).await.unwrap().len(), 3);
+    assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 6); // muted claire is not counted
+
+    // unmute claire again
+    set_muted(&t, claire.id, MuteDuration::NotMuted)
+        .await
+        .unwrap();
+    assert_eq!(claire.id.get_fresh_msg_cnt(&t).await.unwrap(), 3);
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 9); // claire is counted again
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_fresh_msgs_and_muted_until() {
+    let t = TestContext::new_alice().await;
+    let bob = t.create_chat_with_contact("", "bob@g.it").await;
+    receive_msg(&t, &bob).await;
+    assert_eq!(get_chat_msgs(&t, bob.id).await.unwrap().len(), 1);
+
+    // chat is unmuted by default, here and in the following assert(),
+    // we check mainly that the SQL-statements in is_muted() and get_fresh_msgs()
+    // have the same view to the database.
+    assert!(!bob.is_muted());
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+    // test get_fresh_msgs() with mute_until in the future
+    set_muted(
+        &t,
+        bob.id,
+        MuteDuration::Until(SystemTime::now() + Duration::from_secs(3600)),
+    )
+    .await
+    .unwrap();
+    let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+    assert!(bob.is_muted());
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+    // to test get_fresh_msgs() with mute_until in the past,
+    // we need to modify the database directly
+    t.sql
+        .execute(
+            "UPDATE chats SET muted_until=? WHERE id=?;",
+            (time() - 3600, bob.id),
+        )
+        .await
+        .unwrap();
+    let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+    assert!(!bob.is_muted());
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+    // test get_fresh_msgs() with "forever" mute_until
+    set_muted(&t, bob.id, MuteDuration::Forever).await.unwrap();
+    let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+    assert!(bob.is_muted());
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+
+    // to test get_fresh_msgs() with invalid mute_until (everything < -1),
+    // that results in "muted forever" by definition.
+    t.sql
+        .execute("UPDATE chats SET muted_until=-2 WHERE id=?;", (bob.id,))
+        .await
+        .unwrap();
+    let bob = Chat::load_from_db(&t, bob.id).await.unwrap();
+    assert!(!bob.is_muted());
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_muted_context() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 0);
+    t.set_config(Config::IsMuted, Some("1")).await?;
+    let chat = t.create_chat_with_contact("", "bob@g.it").await;
+    receive_msg(&t, &chat).await;
+
+    // muted contexts should still show dimmed badge counters eg. in the sidebars,
+    // (same as muted chats show dimmed badge counters in the chatlist)
+    // therefore the fresh messages count should not be affected.
+    assert_eq!(t.get_fresh_msgs().await.unwrap().len(), 1);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_blobdir_exists() {
+    let tmp = tempfile::tempdir().unwrap();
+    let dbfile = tmp.path().join("db.sqlite");
+    Context::new(&dbfile, 1, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
+    let blobdir = tmp.path().join("db.sqlite-blobs");
+    assert!(blobdir.is_dir());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_wrong_blogdir() {
+    let tmp = tempfile::tempdir().unwrap();
+    let dbfile = tmp.path().join("db.sqlite");
+    let blobdir = tmp.path().join("db.sqlite-blobs");
+    tokio::fs::write(&blobdir, b"123").await.unwrap();
+    let res = Context::new(&dbfile, 1, Events::new(), StockStrings::new()).await;
+    assert!(res.is_err());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_sqlite_parent_not_exists() {
+    let tmp = tempfile::tempdir().unwrap();
+    let subdir = tmp.path().join("subdir");
+    let dbfile = subdir.join("db.sqlite");
+    let dbfile2 = dbfile.clone();
+    Context::new(&dbfile, 1, Events::new(), StockStrings::new())
+        .await
+        .unwrap();
+    assert!(subdir.is_dir());
+    assert!(dbfile2.is_file());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_with_empty_blobdir() {
+    let tmp = tempfile::tempdir().unwrap();
+    let dbfile = tmp.path().join("db.sqlite");
+    let blobdir = PathBuf::new();
+    let res = Context::with_blobdir(
+        dbfile,
+        blobdir,
+        1,
+        Events::new(),
+        StockStrings::new(),
+        Default::default(),
+    );
+    assert!(res.is_err());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_with_blobdir_not_exists() {
+    let tmp = tempfile::tempdir().unwrap();
+    let dbfile = tmp.path().join("db.sqlite");
+    let blobdir = tmp.path().join("blobs");
+    let res = Context::with_blobdir(
+        dbfile,
+        blobdir,
+        1,
+        Events::new(),
+        StockStrings::new(),
+        Default::default(),
+    );
+    assert!(res.is_err());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn no_crashes_on_context_deref() {
+    let t = TestContext::new().await;
+    std::mem::drop(t);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_info() {
+    let t = TestContext::new().await;
+
+    let info = t.get_info().await.unwrap();
+    assert!(info.contains_key("database_dir"));
+}
+
+#[test]
+fn test_get_info_no_context() {
+    let info = get_info();
+    assert!(info.contains_key("deltachat_core_version"));
+    assert!(!info.contains_key("database_dir"));
+    assert_eq!(info.get("level").unwrap(), "awesome");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_info_completeness() {
+    // For easier debugging,
+    // get_info() shall return all important information configurable by the Config-values.
+    //
+    // There are exceptions for Config-values considered to be unimportant,
+    // too sensitive or summarized in another item.
+    let skip_from_get_info = vec![
+        "addr",
+        "displayname",
+        "imap_certificate_checks",
+        "mail_server",
+        "mail_user",
+        "mail_pw",
+        "mail_port",
+        "mail_security",
+        "notify_about_wrong_pw",
+        "self_reporting_id",
+        "selfstatus",
+        "send_server",
+        "send_user",
+        "send_pw",
+        "send_port",
+        "send_security",
+        "server_flags",
+        "skip_start_messages",
+        "smtp_certificate_checks",
+        "proxy_url",      // May contain passwords, don't leak it to the logs.
+        "socks5_enabled", // SOCKS5 options are deprecated.
+        "socks5_host",
+        "socks5_port",
+        "socks5_user",
+        "socks5_password",
+        "key_id",
+        "webxdc_integration",
+        "device_token",
+        "encrypted_device_token",
+    ];
+    let t = TestContext::new().await;
+    let info = t.get_info().await.unwrap();
+    for key in Config::iter() {
+        let key: String = key.to_string();
+        if !skip_from_get_info.contains(&&*key)
+            && !key.starts_with("configured")
+            && !key.starts_with("sys.")
+        {
+            assert!(
+                info.contains_key(&*key),
+                "'{key}' missing in get_info() output"
+            );
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_search_msgs() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let self_talk = ChatId::create_for_contact(&alice, ContactId::SELF).await?;
+    let chat = alice
+        .create_chat_with_contact("Bob", "bob@example.org")
+        .await;
+
+    // Global search finds nothing.
+    let res = alice.search_msgs(None, "foo").await?;
+    assert!(res.is_empty());
+
+    // Search in chat with Bob finds nothing.
+    let res = alice.search_msgs(Some(chat.id), "foo").await?;
+    assert!(res.is_empty());
+
+    // Add messages to chat with Bob.
+    let mut msg1 = Message::new_text("foobar".to_string());
+    send_msg(&alice, chat.id, &mut msg1).await?;
+
+    let mut msg2 = Message::new_text("barbaz".to_string());
+    send_msg(&alice, chat.id, &mut msg2).await?;
+
+    alice.send_text(chat.id, "Δ-Chat").await;
+
+    // Global search with a part of text finds the message.
+    let res = alice.search_msgs(None, "ob").await?;
+    assert_eq!(res.len(), 1);
+
+    // Global search for "bar" matches both "foobar" and "barbaz".
+    let res = alice.search_msgs(None, "bar").await?;
+    assert_eq!(res.len(), 2);
+
+    // Message added later is returned first.
+    assert_eq!(res.first(), Some(&msg2.id));
+    assert_eq!(res.get(1), Some(&msg1.id));
+
+    // Search is case-insensitive.
+    for chat_id in [None, Some(chat.id)] {
+        let res = alice.search_msgs(chat_id, "δ-chat").await?;
+        assert_eq!(res.len(), 1);
+    }
+
+    // Global search with longer text does not find any message.
+    let res = alice.search_msgs(None, "foobarbaz").await?;
+    assert!(res.is_empty());
+
+    // Search for random string finds nothing.
+    let res = alice.search_msgs(None, "abc").await?;
+    assert!(res.is_empty());
+
+    // Search in chat with Bob finds the message.
+    let res = alice.search_msgs(Some(chat.id), "foo").await?;
+    assert_eq!(res.len(), 1);
+
+    // Search in Saved Messages does not find the message.
+    let res = alice.search_msgs(Some(self_talk), "foo").await?;
+    assert!(res.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_search_unaccepted_requests() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    receive_imf(
+        &t,
+        b"From: BobBar <bob@example.org>\n\
+                 To: alice@example.org\n\
+                 Subject: foo\n\
+                 Message-ID: <msg1234@example.org>\n\
+                 Chat-Version: 1.0\n\
+                 Date: Tue, 25 Oct 2022 13:37:00 +0000\n\
+                 \n\
+                 hello bob, foobar test!\n",
+        false,
+    )
+    .await?;
+    let chat_id = t.get_last_msg().await.get_chat_id();
+    let chat = Chat::load_from_db(&t, chat_id).await?;
+    assert_eq!(chat.get_type(), Chattype::Single);
+    assert!(chat.is_contact_request());
+
+    assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 1);
+    assert_eq!(
+        Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
+        1
+    );
+    assert_eq!(t.search_msgs(None, "foobar").await?.len(), 1);
+    assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 1);
+
+    chat_id.block(&t).await?;
+
+    assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 0);
+    assert_eq!(
+        Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
+        0
+    );
+    assert_eq!(t.search_msgs(None, "foobar").await?.len(), 0);
+    assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 0);
+
+    let contact_ids = get_chat_contacts(&t, chat_id).await?;
+    Contact::unblock(&t, *contact_ids.first().unwrap()).await?;
+
+    assert_eq!(Chatlist::try_load(&t, 0, None, None).await?.len(), 1);
+    assert_eq!(
+        Chatlist::try_load(&t, 0, Some("BobBar"), None).await?.len(),
+        1
+    );
+    assert_eq!(t.search_msgs(None, "foobar").await?.len(), 1);
+    assert_eq!(t.search_msgs(Some(chat_id), "foobar").await?.len(), 1);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_limit_search_msgs() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let chat = alice
+        .create_chat_with_contact("Bob", "bob@example.org")
+        .await;
+
+    // Add 999 messages
+    let mut msg = Message::new_text("foobar".to_string());
+    for _ in 0..999 {
+        send_msg(&alice, chat.id, &mut msg).await?;
+    }
+    let res = alice.search_msgs(None, "foo").await?;
+    assert_eq!(res.len(), 999);
+
+    // Add one more message, no limit yet
+    send_msg(&alice, chat.id, &mut msg).await?;
+    let res = alice.search_msgs(None, "foo").await?;
+    assert_eq!(res.len(), 1000);
+
+    // Add one more message, that one is truncated then
+    send_msg(&alice, chat.id, &mut msg).await?;
+    let res = alice.search_msgs(None, "foo").await?;
+    assert_eq!(res.len(), 1000);
+
+    // In-chat should not be not limited
+    let res = alice.search_msgs(Some(chat.id), "foo").await?;
+    assert_eq!(res.len(), 1001);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_check_passphrase() -> Result<()> {
+    let dir = tempdir()?;
+    let dbfile = dir.path().join("db.sqlite");
+
+    let context = ContextBuilder::new(dbfile.clone())
+        .with_id(1)
+        .build()
+        .await
+        .context("failed to create context")?;
+    assert_eq!(context.open("foo".to_string()).await?, true);
+    assert_eq!(context.is_open().await, true);
+    drop(context);
+
+    let context = ContextBuilder::new(dbfile)
+        .with_id(2)
+        .build()
+        .await
+        .context("failed to create context")?;
+    assert_eq!(context.is_open().await, false);
+    assert_eq!(context.check_passphrase("bar".to_string()).await?, false);
+    assert_eq!(context.open("false".to_string()).await?, false);
+    assert_eq!(context.open("foo".to_string()).await?, true);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_context_change_passphrase() -> Result<()> {
+    let dir = tempdir()?;
+    let dbfile = dir.path().join("db.sqlite");
+
+    let context = ContextBuilder::new(dbfile)
+        .with_id(1)
+        .build()
+        .await
+        .context("failed to create context")?;
+    assert_eq!(context.open("foo".to_string()).await?, true);
+    assert_eq!(context.is_open().await, true);
+
+    context
+        .set_config(Config::Addr, Some("alice@example.org"))
+        .await?;
+
+    context
+        .change_passphrase("bar".to_string())
+        .await
+        .context("Failed to change passphrase")?;
+
+    assert_eq!(
+        context.get_config(Config::Addr).await?.unwrap(),
+        "alice@example.org"
+    );
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ongoing() -> Result<()> {
+    let context = TestContext::new().await;
+
+    // No ongoing process allocated.
+    assert!(context.shall_stop_ongoing().await);
+
+    let receiver = context.alloc_ongoing().await?;
+
+    // Cannot allocate another ongoing process while the first one is running.
+    assert!(context.alloc_ongoing().await.is_err());
+
+    // Stop signal is not sent yet.
+    assert!(receiver.try_recv().is_err());
+
+    assert!(!context.shall_stop_ongoing().await);
+
+    // Send the stop signal.
+    context.stop_ongoing().await;
+
+    // Receive stop signal.
+    receiver.recv().await?;
+
+    assert!(context.shall_stop_ongoing().await);
+
+    // Ongoing process is still running even though stop signal was received,
+    // so another one cannot be allocated.
+    assert!(context.alloc_ongoing().await.is_err());
+
+    context.free_ongoing().await;
+
+    // No ongoing process allocated, should have been stopped already.
+    assert!(context.shall_stop_ongoing().await);
+
+    // Another ongoing process can be allocated now.
+    let _receiver = context.alloc_ongoing().await?;
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_next_msgs() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+
+    let alice_chat = alice.create_chat(&bob).await;
+
+    assert!(alice.get_next_msgs().await?.is_empty());
+    assert!(bob.get_next_msgs().await?.is_empty());
+
+    let sent_msg = alice.send_text(alice_chat.id, "Hi Bob").await;
+    let received_msg = bob.recv_msg(&sent_msg).await;
+
+    let bob_next_msg_ids = bob.get_next_msgs().await?;
+    assert_eq!(bob_next_msg_ids.len(), 1);
+    assert_eq!(bob_next_msg_ids.first(), Some(&received_msg.id));
+
+    bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
+        .await?;
+    assert!(bob.get_next_msgs().await?.is_empty());
+
+    // Next messages include self-sent messages.
+    let alice_next_msg_ids = alice.get_next_msgs().await?;
+    assert_eq!(alice_next_msg_ids.len(), 1);
+    assert_eq!(alice_next_msg_ids.first(), Some(&sent_msg.sender_msg_id));
+
+    alice
+        .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())
+        .await?;
+    assert!(alice.get_next_msgs().await?.is_empty());
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_draft_self_report() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+
+    let chat_id = alice.draft_self_report().await?;
+    let msg = get_chat_msg(&alice, chat_id, 0, 1).await;
+    assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled);
+
+    let chat = Chat::load_from_db(&alice, chat_id).await?;
+    assert!(chat.is_protected());
+
+    let mut draft = chat_id.get_draft(&alice).await?.unwrap();
+    assert!(draft.text.starts_with("core_version"));
+
+    // Test that sending into the protected chat works:
+    let _sent = alice.send_msg(chat_id, &mut draft).await;
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_cache_is_cleared_when_io_is_started() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    assert_eq!(
+        alice.get_config(Config::ShowEmails).await?,
+        Some("2".to_string())
+    );
+
+    // Change the config circumventing the cache
+    // This simulates what the notification plugin on iOS might do
+    // because it runs in a different process
+    alice
+        .sql
+        .execute(
+            "INSERT OR REPLACE INTO config (keyname, value) VALUES ('show_emails', '0')",
+            (),
+        )
+        .await?;
+
+    // Alice's Delta Chat doesn't know about it yet:
+    assert_eq!(
+        alice.get_config(Config::ShowEmails).await?,
+        Some("2".to_string())
+    );
+
+    // Starting IO will fail of course because no server settings are configured,
+    // but it should invalidate the caches:
+    alice.start_io().await;
+
+    assert_eq!(
+        alice.get_config(Config::ShowEmails).await?,
+        Some("0".to_string())
+    );
+
+    Ok(())
+}

src/debug_logging.rs

@@ -60,9 +60,11 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                         "time": time,
                     }),
                     info: None,
+                    href: None,
                     summary: None,
                     document: None,
                     uid: None,
+                    notify: None,
                 },
                 time,
             )
@@ -98,7 +100,9 @@ pub async fn maybe_set_logging_xdc(
         context,
         msg.get_viewtype(),
         chat_id,
-        msg.param.get_path(Param::File, context).unwrap_or_default(),
+        msg.param
+            .get_path(Param::Filename, context)
+            .unwrap_or_default(),
         msg.get_id(),
     )
     .await?;
@@ -111,11 +115,11 @@ pub async fn maybe_set_logging_xdc_inner(
     context: &Context,
     viewtype: Viewtype,
     chat_id: ChatId,
-    file: Option<PathBuf>,
+    filename: Option<PathBuf>,
     msg_id: MsgId,
 ) -> anyhow::Result<()> {
     if viewtype == Viewtype::Webxdc {
-        if let Some(file) = file {
+        if let Some(file) = filename {
             if let Some(file_name) = file.file_name().and_then(|name| name.to_str()) {
                 if file_name.starts_with("debug_logging")
                     && file_name.ends_with(".xdc")

src/decrypt.rs

@@ -182,7 +182,7 @@ pub(crate) async fn get_autocrypt_peerstate(
             // if the fingerprint is verified.
             peerstate = Peerstate::from_verified_fingerprint_or_addr(
                 context,
-                &header.public_key.fingerprint(),
+                &header.public_key.dc_fingerprint(),
                 from,
             )
             .await?;

src/download.rs

@@ -3,7 +3,7 @@
 use std::cmp::max;
 use std::collections::BTreeMap;
 
-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 
@@ -201,7 +201,11 @@ impl Session {
             bail!("Attempt to fetch UID 0");
         }
 
-        self.select_with_uidvalidity(context, folder).await?;
+        let create = false;
+        let folder_exists = self
+            .select_with_uidvalidity(context, folder, create)
+            .await?;
+        ensure!(folder_exists, "No folder {folder}");
 
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
@@ -436,11 +440,11 @@ mod tests {
         let file = alice.get_blobdir().join("minimal.xdc");
         tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?;
         let mut instance = Message::new(Viewtype::File);
-        instance.set_file(file.to_str().unwrap(), None);
+        instance.set_file_and_deduplicate(&alice, &file, None, None)?;
         let _sent1 = alice.send_msg(chat_id, &mut instance).await;
 
         alice
-            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#, "d")
+            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#)
             .await?;
         alice.flush_status_updates().await?;
         let sent2 = alice.pop_sent_msg().await;

src/e2ee.rs

@@ -40,20 +40,17 @@ impl EncryptHelper {
 
     /// Determines if we can and should encrypt.
     ///
-    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
-    /// of peerstates should prefer encryption. Own preference is counted equally to peer
-    /// preferences, even if message copy is not sent to self.
-    ///
     /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
     /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
     ///
     /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
-    pub fn should_encrypt(
+    pub(crate) async fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, String)],
     ) -> Result<bool> {
+        let is_chatmail = context.is_chatmail().await?;
         let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
             1
         } else {
@@ -64,10 +61,15 @@ impl EncryptHelper {
                 Some(peerstate) => {
                     let prefer_encrypt = peerstate.prefer_encrypt;
                     info!(context, "Peerstate for {addr:?} is {prefer_encrypt}.");
-                    match peerstate.prefer_encrypt {
-                        EncryptPreference::NoPreference | EncryptPreference::Reset => {}
-                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
-                    };
+                    if match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference | EncryptPreference::Reset => {
+                            (peerstate.prefer_encrypt != EncryptPreference::Reset || is_chatmail)
+                                && self.prefer_encrypt == EncryptPreference::Mutual
+                        }
+                        EncryptPreference::Mutual => true,
+                    } {
+                        prefer_encrypt_count += 1;
+                    }
                 }
                 None => {
                     let msg = format!("Peerstate for {addr:?} missing, cannot encrypt");
@@ -170,9 +172,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::send_text_msg;
     use crate::key::DcKey;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
+    use crate::receive_imf::receive_imf;
     use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
 
     mod ensure_secret_key_exists {
@@ -303,12 +307,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             last_seen_autocrypt: 14,
             prefer_encrypt,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint()),
+            public_key_fingerprint: Some(pub_key.dc_fingerprint()),
             gossip_key: Some(pub_key.clone()),
             gossip_timestamp: 15,
-            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key_fingerprint: Some(pub_key.dc_fingerprint()),
             verified_key: Some(pub_key.clone()),
-            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key_fingerprint: Some(pub_key.dc_fingerprint()),
             verifier: None,
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
@@ -320,29 +324,109 @@ Sent with my Delta Chat Messenger: https://delta.chat";
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_should_encrypt() {
+    async fn test_should_encrypt() -> Result<()> {
         let t = TestContext::new_alice().await;
+        assert!(t.get_config_bool(Config::E2eeEnabled).await?);
         let encrypt_helper = EncryptHelper::new(&t).await.unwrap();
 
-        // test with EncryptPreference::NoPreference:
-        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
         let ps = new_peerstates(EncryptPreference::NoPreference);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).await?);
+        // Own preference is `Mutual` and we have the peer's key.
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).await?);
 
-        // test with EncryptPreference::Reset
         let ps = new_peerstates(EncryptPreference::Reset);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).await?);
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).await?);
 
-        // test with EncryptPreference::Mutual (self is also Mutual)
         let ps = new_peerstates(EncryptPreference::Mutual);
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).unwrap());
-        assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).await?);
+        assert!(encrypt_helper.should_encrypt(&t, false, &ps).await?);
 
         // test with missing peerstate
         let ps = vec![(None, "bob@foo.bar".to_string())];
-        assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
-        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t, true, &ps).await.is_err());
+        assert!(!encrypt_helper.should_encrypt(&t, false, &ps).await?);
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_should_encrypt_e2ee_disabled() -> Result<()> {
+        let t = &TestContext::new_alice().await;
+        t.set_config_bool(Config::E2eeEnabled, false).await?;
+        let encrypt_helper = EncryptHelper::new(t).await.unwrap();
+
+        let ps = new_peerstates(EncryptPreference::NoPreference);
+        assert!(!encrypt_helper.should_encrypt(t, false, &ps).await?);
+
+        let ps = new_peerstates(EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(t, true, &ps).await?);
+
+        let mut ps = new_peerstates(EncryptPreference::Mutual);
+        // Own preference is `NoPreference` and there's no majority with `Mutual`.
+        assert!(!encrypt_helper.should_encrypt(t, false, &ps).await?);
+        // Now the majority wants to encrypt. Let's encrypt, anyway there are other cases when we
+        // can't send unencrypted, e.g. protected groups.
+        ps.push(ps[0].clone());
+        assert!(encrypt_helper.should_encrypt(t, false, &ps).await?);
+
+        // Test with missing peerstate.
+        let ps = vec![(None, "bob@foo.bar".to_string())];
+        assert!(encrypt_helper.should_encrypt(t, true, &ps).await.is_err());
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_chatmail_prefers_to_encrypt() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+        bob.set_config_bool(Config::IsChatmail, true).await?;
+
+        let bob_chat_id = tcm
+            .send_recv_accept(alice, bob, "Hello from DC")
+            .await
+            .chat_id;
+        receive_imf(
+            bob,
+            b"From: alice@example.org\n\
+            To: bob@example.net\n\
+            Message-ID: <2222@example.org>\n\
+            Date: Sun, 22 Mar 3000 22:37:58 +0000\n\
+            \n\
+            Hello from another MUA\n",
+            false,
+        )
+        .await?;
+        send_text_msg(bob, bob_chat_id, "hi".to_string()).await?;
+        let sent_msg = bob.pop_sent_msg().await;
+        let msg = Message::load_from_db(bob, sent_msg.sender_msg_id).await?;
+        assert!(msg.get_showpadlock());
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_chatmail_can_send_unencrypted() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let bob = &tcm.bob().await;
+        bob.set_config_bool(Config::IsChatmail, true).await?;
+        let bob_chat_id = receive_imf(
+            bob,
+            b"From: alice@example.org\n\
+            To: bob@example.net\n\
+            Message-ID: <2222@example.org>\n\
+            Date: Sun, 22 Mar 3000 22:37:58 +0000\n\
+            \n\
+            Hello\n",
+            false,
+        )
+        .await?
+        .unwrap()
+        .chat_id;
+        bob_chat_id.accept(bob).await?;
+        send_text_msg(bob, bob_chat_id, "hi".to_string()).await?;
+        let sent_msg = bob.pop_sent_msg().await;
+        let msg = Message::load_from_db(bob, sent_msg.sender_msg_id).await?;
+        assert!(!msg.get_showpadlock());
+        Ok(())
     }
 }

src/ephemeral.rs

@@ -84,7 +84,6 @@ use crate::location;
 use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::sql::{self, params_iter};
 use crate::stock_str;
 use crate::tools::{duration_to_str, time, SystemTime};
 
@@ -329,23 +328,44 @@ pub(crate) async fn start_ephemeral_timers_msgids(
     msg_ids: &[MsgId],
 ) -> Result<()> {
     let now = time();
-    let count = context
+    let should_interrupt =
+    context
+        .sql
+        .transaction(move |transaction| {
+            let mut should_interrupt = false;
+            let mut stmt =
+                transaction.prepare(
+                    "UPDATE msgs SET ephemeral_timestamp = ?1 + ephemeral_timer
+                     WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?1 + ephemeral_timer) AND ephemeral_timer > 0
+                     AND id=?2")?;
+            for msg_id in msg_ids {
+                should_interrupt |= stmt.execute((now, msg_id))? > 0;
+            }
+            Ok(should_interrupt)
+        }).await?;
+    if should_interrupt {
+        context.scheduler.interrupt_ephemeral_task().await;
+    }
+    Ok(())
+}
+
+/// Starts ephemeral timer for all messages in the chat.
+///
+/// This should be called when chat is marked as noticed.
+pub(crate) async fn start_chat_ephemeral_timers(context: &Context, chat_id: ChatId) -> Result<()> {
+    let now = time();
+    let should_interrupt = context
         .sql
         .execute(
-            &format!(
-                "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
-         WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
-         AND id IN ({})",
-                sql::repeat_vars(msg_ids.len())
-            ),
-            rusqlite::params_from_iter(
-                std::iter::once(&now as &dyn crate::sql::ToSql)
-                    .chain(std::iter::once(&now as &dyn crate::sql::ToSql))
-                    .chain(params_iter(msg_ids)),
-            ),
+            "UPDATE msgs SET ephemeral_timestamp = ?1 + ephemeral_timer
+             WHERE chat_id = ?2
+             AND ephemeral_timer > 0
+             AND (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?1 + ephemeral_timer)",
+            (now, chat_id),
         )
-        .await?;
-    if count > 0 {
+        .await?
+        > 0;
+    if should_interrupt {
         context.scheduler.interrupt_ephemeral_task().await;
     }
     Ok(())
@@ -482,7 +502,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
         }
 
         for modified_chat_id in modified_chat_ids {
-            context.emit_msgs_changed(modified_chat_id, MsgId::new(0));
+            context.emit_msgs_changed_without_msg_id(modified_chat_id);
         }
 
         for msg_id in webxdc_deleted {
@@ -693,736 +713,4 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 }
 
 #[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::Config;
-    use crate::download::DownloadState;
-    use crate::location;
-    use crate::message::markseen_msgs;
-    use crate::receive_imf::receive_imf;
-    use crate::test_utils::{TestContext, TestContextManager};
-    use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE;
-    use crate::{
-        chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
-        tools::IsNoneOrEmpty,
-    };
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_stock_ephemeral_messages() {
-        let context = TestContext::new().await;
-
-        assert_eq!(
-            stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
-            "You disabled message deletion timer."
-        );
-
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 1 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1 s."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 30 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 30 s."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 60 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1 minute."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 90 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1.5 minutes."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 30 * 60 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 30 minutes."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 60 * 60 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1 hour."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled { duration: 5400 },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1.5 hours."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 2 * 60 * 60
-                },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 2 hours."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 24 * 60 * 60
-                },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1 day."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 2 * 24 * 60 * 60
-                },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 2 days."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 7 * 24 * 60 * 60
-                },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 1 week."
-        );
-        assert_eq!(
-            stock_ephemeral_timer_changed(
-                &context,
-                Timer::Enabled {
-                    duration: 4 * 7 * 24 * 60 * 60
-                },
-                ContactId::SELF
-            )
-            .await,
-            "You set message deletion timer to 4 weeks."
-        );
-    }
-
-    /// Test enabling and disabling ephemeral timer remotely.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_enable_disable() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled
-        );
-
-        Ok(())
-    }
-
-    /// Test that enabling ephemeral timer in unpromoted group does not send a message.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_unpromoted() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        let chat_id =
-            create_group_chat(&alice, ProtectionStatus::Unprotected, "Group name").await?;
-
-        // Group is unpromoted, the timer can be changed without sending a message.
-        assert!(chat_id.is_unpromoted(&alice).await?);
-        chat_id
-            .set_ephemeral_timer(&alice, Timer::Enabled { duration: 60 })
-            .await?;
-        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
-        assert!(sent.is_none());
-        assert_eq!(
-            chat_id.get_ephemeral_timer(&alice).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Promote the group.
-        send_text_msg(&alice, chat_id, "hi!".to_string()).await?;
-        assert!(chat_id.is_promoted(&alice).await?);
-        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
-        assert!(sent.is_some());
-
-        chat_id
-            .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
-            .await?;
-        let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
-        assert!(sent.is_some());
-        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
-
-        Ok(())
-    }
-
-    /// Test that timer is enabled even if the message explicitly enabling the timer is lost.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_enable_lost() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        // Alice enables the timer.
-        chat_alice
-            .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-        // The message enabling the timer is lost.
-        let _sent = alice.pop_sent_msg().await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled,
-        );
-
-        // Alice sends a text message.
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-
-        // Bob receives text message and enables the timer, even though explicit timer update was
-        // lost previously.
-        bob.recv_msg(&sent).await;
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        Ok(())
-    }
-
-    /// Test that Alice replying to the chat without a timer at the same time as Bob enables the
-    /// timer does not result in disabling the timer on the Bob's side.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_timer_rollback() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let chat_alice = alice.create_chat(&bob).await.id;
-        let chat_bob = bob.create_chat(&alice).await.id;
-
-        // Alice sends message to Bob
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-
-        // Alice sends second message to Bob, with no timer
-        let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
-
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Disabled
-        );
-
-        // Bob sets ephemeral timer and sends a message about timer change
-        chat_bob
-            .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
-            .await?;
-        let sent_timer_change = bob.pop_sent_msg().await;
-
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Bob receives message from Alice.
-        // Alice message has no timer. However, Bob should not disable timer,
-        // because Alice replies to old message.
-        bob.recv_msg(&sent).await;
-
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Disabled
-        );
-        assert_eq!(
-            chat_bob.get_ephemeral_timer(&bob.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Alice receives message from Bob
-        alice.recv_msg(&sent_timer_change).await;
-
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Enabled { duration: 60 }
-        );
-
-        // Bob disables the chat timer.
-        // Note that the last message in the Bob's chat is from Alice and has no timer,
-        // but the chat timer is enabled.
-        chat_bob
-            .set_ephemeral_timer(&bob.ctx, Timer::Disabled)
-            .await?;
-        alice.recv_msg(&bob.pop_sent_msg().await).await;
-        assert_eq!(
-            chat_alice.get_ephemeral_timer(&alice.ctx).await?,
-            Timer::Disabled
-        );
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_delete_msgs() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let self_chat = t.get_self_chat().await;
-
-        assert_eq!(next_expiration_timestamp(&t).await, None);
-
-        t.send_text(self_chat.id, "Saved message, which we delete manually")
-            .await;
-        let msg = t.get_last_msg_in(self_chat.id).await;
-        msg.id.trash(&t, false).await?;
-        check_msg_is_deleted(&t, &self_chat, msg.id).await;
-
-        self_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
-            .await
-            .unwrap();
-
-        // Send a saved message which will be deleted after 3600s
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
-            .await
-            .unwrap();
-
-        // Set DeleteDeviceAfter to 1800s. Then send a saved message which will
-        // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
-        t.set_config(Config::DeleteDeviceAfter, Some("1800"))
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(self_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
-            .await
-            .unwrap();
-
-        // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
-        let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(
-            &t,
-            msg.sender_msg_id,
-            &bob_chat,
-            now + 1799,
-            // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
-            // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
-            time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
-        )
-        .await
-        .unwrap();
-
-        // Enable ephemeral messages with Bob -> message will be deleted after 60s.
-        // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
-        bob_chat
-            .id
-            .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
-            .await?;
-
-        let now = time();
-        let msg = t.send_text(bob_chat.id, "Message text").await;
-
-        check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
-            .await
-            .unwrap();
-
-        Ok(())
-    }
-
-    async fn check_msg_will_be_deleted(
-        t: &TestContext,
-        msg_id: MsgId,
-        chat: &Chat,
-        not_deleted_at: i64,
-        deleted_at: i64,
-    ) -> Result<()> {
-        let next_expiration = next_expiration_timestamp(t).await.unwrap();
-
-        assert!(next_expiration > not_deleted_at);
-        delete_expired_messages(t, not_deleted_at).await?;
-
-        let loaded = Message::load_from_db(t, msg_id).await?;
-        assert!(!loaded.text.is_empty());
-        assert_eq!(loaded.chat_id, chat.id);
-
-        assert!(next_expiration < deleted_at);
-        delete_expired_messages(t, deleted_at).await?;
-        t.evtracker
-            .get_matching(|evt| {
-                if let EventType::MsgDeleted {
-                    msg_id: event_msg_id,
-                    ..
-                } = evt
-                {
-                    *event_msg_id == msg_id
-                } else {
-                    false
-                }
-            })
-            .await;
-
-        let loaded = Message::load_from_db_optional(t, msg_id).await?;
-        assert!(loaded.is_none());
-
-        // Check that the msg was deleted locally.
-        check_msg_is_deleted(t, chat, msg_id).await;
-
-        Ok(())
-    }
-
-    async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
-        let chat_items = chat::get_chat_msgs(t, chat.id).await.unwrap();
-        // Check that the chat is empty except for possibly info messages:
-        for item in &chat_items {
-            if let ChatItem::Message { msg_id } = item {
-                let msg = Message::load_from_db(t, *msg_id).await.unwrap();
-                assert!(msg.is_info())
-            }
-        }
-
-        // Check that if there is a message left, the text and metadata are gone
-        if let Ok(msg) = Message::load_from_db(t, msg_id).await {
-            assert_eq!(msg.from_id, ContactId::UNDEFINED);
-            assert_eq!(msg.to_id, ContactId::UNDEFINED);
-            assert_eq!(msg.text, "");
-            let rawtxt: Option<String> = t
-                .sql
-                .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", (msg_id,))
-                .await
-                .unwrap();
-            assert!(rawtxt.is_none_or_empty(), "{rawtxt:?}");
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_delete_expired_imap_messages() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        const HOUR: i64 = 60 * 60;
-        let now = time();
-        for (id, timestamp, ephemeral_timestamp) in &[
-            (900, now - 2 * HOUR, 0),
-            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
-            (1010, now - 23 * HOUR, 0),
-            (1020, now - 21 * HOUR, 0),
-            (1030, now - 19 * HOUR, 0),
-            (2000, now - 18 * HOUR, now - HOUR),
-            (2020, now - 17 * HOUR, now + HOUR),
-            (3000, now + HOUR, 0),
-        ] {
-            let message_id = id.to_string();
-            t.sql
-                   .execute(
-                       "INSERT INTO msgs (id, rfc724_mid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
-                       (id, &message_id, timestamp, ephemeral_timestamp),
-                   )
-                   .await?;
-            t.sql
-                   .execute(
-                       "INSERT INTO imap (rfc724_mid, folder, uid, target) VALUES (?,'INBOX',?, 'INBOX');",
-                       (&message_id, id),
-                   )
-                   .await?;
-        }
-
-        async fn test_marked_for_deletion(context: &Context, id: u32) -> Result<()> {
-            assert_eq!(
-                context
-                    .sql
-                    .count(
-                        "SELECT COUNT(*) FROM imap WHERE target='' AND rfc724_mid=?",
-                        (id.to_string(),),
-                    )
-                    .await?,
-                1
-            );
-            Ok(())
-        }
-
-        async fn remove_uid(context: &Context, id: u32) -> Result<()> {
-            context
-                .sql
-                .execute("DELETE FROM imap WHERE rfc724_mid=?", (id.to_string(),))
-                .await?;
-            Ok(())
-        }
-
-        // This should mark message 2000 for deletion.
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 2000).await?;
-        remove_uid(&t, 2000).await?;
-        // No other messages are marked for deletion.
-        assert_eq!(
-            t.sql
-                .count("SELECT COUNT(*) FROM imap WHERE target=''", ())
-                .await?,
-            0
-        );
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1000).await?;
-
-        MsgId::new(1000)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        t.sql
-            .execute("UPDATE imap SET target=folder WHERE rfc724_mid='1000'", ())
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1000).await?; // Delete downloadable anyway.
-        remove_uid(&t, 1000).await?;
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 1010).await?;
-        t.sql
-            .execute("UPDATE imap SET target=folder WHERE rfc724_mid='1010'", ())
-            .await?;
-
-        MsgId::new(1010)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        delete_expired_imap_messages(&t).await?;
-        // Keep downloadable for now.
-        assert_eq!(
-            t.sql
-                .count("SELECT COUNT(*) FROM imap WHERE target=''", ())
-                .await?,
-            0
-        );
-
-        t.set_config(Config::DeleteServerAfter, Some("1")).await?;
-        delete_expired_imap_messages(&t).await?;
-        test_marked_for_deletion(&t, 3000).await?;
-
-        Ok(())
-    }
-
-    // Regression test for a bug in the timer rollback protection.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_timer_references() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        // Message with Message-ID <first@example.com> and no timer is received.
-        receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <first@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
-                    \n\
-                    hello\n",
-            false,
-        )
-        .await?;
-
-        let msg = alice.get_last_msg().await;
-        let chat_id = msg.chat_id;
-        assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
-
-        // Message with Message-ID <second@example.com> is received.
-        receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <second@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:11:00 +0000\n\
-                    Ephemeral-Timer: 60\n\
-                    \n\
-                    second message\n",
-            false,
-        )
-        .await?;
-        assert_eq!(
-            chat_id.get_ephemeral_timer(&alice).await?,
-            Timer::Enabled { duration: 60 }
-        );
-        let msg = alice.get_last_msg().await;
-
-        // Message is deleted when its timer expires.
-        msg.id.trash(&alice, false).await?;
-
-        // Message with Message-ID <third@example.com>, referencing <first@example.com> and
-        // <second@example.com>, is received.  The message <second@example.come> is not in the
-        // database anymore, so the timer should be applied unconditionally without rollback
-        // protection.
-        //
-        // Previously Delta Chat fallen back to using <first@example.com> in this case and
-        // compared received timer value to the timer value of the <first@example.com>. Because
-        // their timer values are the same ("disabled"), Delta Chat assumed that the timer was not
-        // changed explicitly and the change should be ignored.
-        //
-        // The message also contains a quote of the first message to test that only References:
-        // header and not In-Reply-To: is consulted by the rollback protection.
-        receive_imf(
-            &alice,
-            b"From: Bob <bob@example.com>\n\
-                    To: Alice <alice@example.org>\n\
-                    Chat-Version: 1.0\n\
-                    Subject: Subject\n\
-                    Message-ID: <third@example.com>\n\
-                    Date: Sun, 22 Mar 2020 00:12:00 +0000\n\
-                    References: <first@example.com> <second@example.com>\n\
-                    In-Reply-To: <first@example.com>\n\
-                    \n\
-                    > hello\n",
-            false,
-        )
-        .await?;
-
-        let msg = alice.get_last_msg().await;
-        assert_eq!(
-            msg.chat_id.get_ephemeral_timer(&alice).await?,
-            Timer::Disabled
-        );
-
-        Ok(())
-    }
-
-    // Tests that if we are offline for a time longer than the ephemeral timer duration, the message
-    // is deleted from the chat but is still in the "smtp" table, i.e. will be sent upon a
-    // successful reconnection.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_msg_offline() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let chat = alice
-            .create_chat_with_contact("Bob", "bob@example.org")
-            .await;
-        let duration = 60;
-        chat.id
-            .set_ephemeral_timer(&alice, Timer::Enabled { duration })
-            .await?;
-        let mut msg = Message::new_text("hi".to_string());
-        assert!(chat::send_msg_sync(&alice, chat.id, &mut msg)
-            .await
-            .is_err());
-        let stmt = "SELECT COUNT(*) FROM smtp WHERE msg_id=?";
-        assert!(alice.sql.exists(stmt, (msg.id,)).await?);
-        let now = time();
-        check_msg_will_be_deleted(&alice, msg.id, &chat, now, now + i64::from(duration) + 1)
-            .await?;
-        assert!(alice.sql.exists(stmt, (msg.id,)).await?);
-
-        Ok(())
-    }
-
-    /// Tests that POI location is deleted when ephemeral message expires.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_ephemeral_poi_location() -> Result<()> {
-        let mut tcm = TestContextManager::new();
-        let alice = &tcm.alice().await;
-        let bob = &tcm.bob().await;
-
-        let chat = alice.create_chat(bob).await;
-
-        let duration = 60;
-        chat.id
-            .set_ephemeral_timer(alice, Timer::Enabled { duration })
-            .await?;
-        let sent = alice.pop_sent_msg().await;
-        bob.recv_msg(&sent).await;
-
-        let mut poi_msg = Message::new_text("Here".to_string());
-        poi_msg.set_location(10.0, 20.0);
-
-        let alice_sent_message = alice.send_msg(chat.id, &mut poi_msg).await;
-        let bob_received_message = bob.recv_msg(&alice_sent_message).await;
-        markseen_msgs(bob, vec![bob_received_message.id]).await?;
-
-        for account in [alice, bob] {
-            let locations = location::get_range(account, None, None, 0, 0).await?;
-            assert_eq!(locations.len(), 1);
-        }
-
-        SystemTime::shift(Duration::from_secs(100));
-
-        for account in [alice, bob] {
-            delete_expired_messages(account, time()).await?;
-            let locations = location::get_range(account, None, None, 0, 0).await?;
-            assert_eq!(locations.len(), 0);
-        }
-
-        Ok(())
-    }
-
-    /// Tests that `.get_ephemeral_timer()` returns an error for invalid chat ID.
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_get_ephemeral_timer_wrong_chat_id() -> Result<()> {
-        let context = TestContext::new().await;
-        let chat_id = ChatId::new(12345);
-        assert!(chat_id.get_ephemeral_timer(&context).await.is_err());
-
-        Ok(())
-    }
-}
+mod ephemeral_tests;

src/ephemeral/ephemeral_tests.rs

@@ -0,0 +1,781 @@
+use super::*;
+use crate::chat::{marknoticed_chat, set_muted, ChatVisibility, MuteDuration};
+use crate::config::Config;
+use crate::constants::DC_CHAT_ID_ARCHIVED_LINK;
+use crate::download::DownloadState;
+use crate::location;
+use crate::message::markseen_msgs;
+use crate::receive_imf::receive_imf;
+use crate::test_utils::{TestContext, TestContextManager};
+use crate::timesmearing::MAX_SECONDS_TO_LEND_FROM_FUTURE;
+use crate::{
+    chat::{self, create_group_chat, send_text_msg, Chat, ChatItem, ProtectionStatus},
+    tools::IsNoneOrEmpty,
+};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_stock_ephemeral_messages() {
+    let context = TestContext::new().await;
+
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Disabled, ContactId::SELF).await,
+        "You disabled message deletion timer."
+    );
+
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, ContactId::SELF)
+            .await,
+        "You set message deletion timer to 1 s."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, ContactId::SELF)
+            .await,
+        "You set message deletion timer to 30 s."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, ContactId::SELF)
+            .await,
+        "You set message deletion timer to 1 minute."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 90 }, ContactId::SELF)
+            .await,
+        "You set message deletion timer to 1.5 minutes."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled { duration: 30 * 60 },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 30 minutes."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled { duration: 60 * 60 },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 1 hour."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 5400 }, ContactId::SELF)
+            .await,
+        "You set message deletion timer to 1.5 hours."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled {
+                duration: 2 * 60 * 60
+            },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 2 hours."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled {
+                duration: 24 * 60 * 60
+            },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 1 day."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled {
+                duration: 2 * 24 * 60 * 60
+            },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 2 days."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled {
+                duration: 7 * 24 * 60 * 60
+            },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 1 week."
+    );
+    assert_eq!(
+        stock_ephemeral_timer_changed(
+            &context,
+            Timer::Enabled {
+                duration: 4 * 7 * 24 * 60 * 60
+            },
+            ContactId::SELF
+        )
+        .await,
+        "You set message deletion timer to 4 weeks."
+    );
+}
+
+/// Test enabling and disabling ephemeral timer remotely.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_enable_disable() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+
+    let chat_alice = alice.create_chat(&bob).await.id;
+    let chat_bob = bob.create_chat(&alice).await.id;
+
+    chat_alice
+        .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+        .await?;
+    let sent = alice.pop_sent_msg().await;
+    bob.recv_msg(&sent).await;
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    chat_alice
+        .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
+        .await?;
+    let sent = alice.pop_sent_msg().await;
+    bob.recv_msg(&sent).await;
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Disabled
+    );
+
+    Ok(())
+}
+
+/// Test that enabling ephemeral timer in unpromoted group does not send a message.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_unpromoted() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+
+    let chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "Group name").await?;
+
+    // Group is unpromoted, the timer can be changed without sending a message.
+    assert!(chat_id.is_unpromoted(&alice).await?);
+    chat_id
+        .set_ephemeral_timer(&alice, Timer::Enabled { duration: 60 })
+        .await?;
+    let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+    assert!(sent.is_none());
+    assert_eq!(
+        chat_id.get_ephemeral_timer(&alice).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    // Promote the group.
+    send_text_msg(&alice, chat_id, "hi!".to_string()).await?;
+    assert!(chat_id.is_promoted(&alice).await?);
+    let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+    assert!(sent.is_some());
+
+    chat_id
+        .set_ephemeral_timer(&alice.ctx, Timer::Disabled)
+        .await?;
+    let sent = alice.pop_sent_msg_opt(Duration::from_secs(1)).await;
+    assert!(sent.is_some());
+    assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
+
+    Ok(())
+}
+
+/// Test that timer is enabled even if the message explicitly enabling the timer is lost.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_enable_lost() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+
+    let chat_alice = alice.create_chat(&bob).await.id;
+    let chat_bob = bob.create_chat(&alice).await.id;
+
+    // Alice enables the timer.
+    chat_alice
+        .set_ephemeral_timer(&alice.ctx, Timer::Enabled { duration: 60 })
+        .await?;
+    assert_eq!(
+        chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+    // The message enabling the timer is lost.
+    let _sent = alice.pop_sent_msg().await;
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Disabled,
+    );
+
+    // Alice sends a text message.
+    let mut msg = Message::new(Viewtype::Text);
+    chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+    let sent = alice.pop_sent_msg().await;
+
+    // Bob receives text message and enables the timer, even though explicit timer update was
+    // lost previously.
+    bob.recv_msg(&sent).await;
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    Ok(())
+}
+
+/// Test that Alice replying to the chat without a timer at the same time as Bob enables the
+/// timer does not result in disabling the timer on the Bob's side.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_timer_rollback() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let bob = TestContext::new_bob().await;
+
+    let chat_alice = alice.create_chat(&bob).await.id;
+    let chat_bob = bob.create_chat(&alice).await.id;
+
+    // Alice sends message to Bob
+    let mut msg = Message::new(Viewtype::Text);
+    chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+    let sent = alice.pop_sent_msg().await;
+    bob.recv_msg(&sent).await;
+
+    // Alice sends second message to Bob, with no timer
+    let mut msg = Message::new(Viewtype::Text);
+    chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+    let sent = alice.pop_sent_msg().await;
+
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Disabled
+    );
+
+    // Bob sets ephemeral timer and sends a message about timer change
+    chat_bob
+        .set_ephemeral_timer(&bob.ctx, Timer::Enabled { duration: 60 })
+        .await?;
+    let sent_timer_change = bob.pop_sent_msg().await;
+
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    // Bob receives message from Alice.
+    // Alice message has no timer. However, Bob should not disable timer,
+    // because Alice replies to old message.
+    bob.recv_msg(&sent).await;
+
+    assert_eq!(
+        chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+        Timer::Disabled
+    );
+    assert_eq!(
+        chat_bob.get_ephemeral_timer(&bob.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    // Alice receives message from Bob
+    alice.recv_msg(&sent_timer_change).await;
+
+    assert_eq!(
+        chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+        Timer::Enabled { duration: 60 }
+    );
+
+    // Bob disables the chat timer.
+    // Note that the last message in the Bob's chat is from Alice and has no timer,
+    // but the chat timer is enabled.
+    chat_bob
+        .set_ephemeral_timer(&bob.ctx, Timer::Disabled)
+        .await?;
+    alice.recv_msg(&bob.pop_sent_msg().await).await;
+    assert_eq!(
+        chat_alice.get_ephemeral_timer(&alice.ctx).await?,
+        Timer::Disabled
+    );
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_delete_msgs() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    let self_chat = t.get_self_chat().await;
+
+    assert_eq!(next_expiration_timestamp(&t).await, None);
+
+    t.send_text(self_chat.id, "Saved message, which we delete manually")
+        .await;
+    let msg = t.get_last_msg_in(self_chat.id).await;
+    msg.id.trash(&t, false).await?;
+    check_msg_is_deleted(&t, &self_chat, msg.id).await;
+
+    self_chat
+        .id
+        .set_ephemeral_timer(&t, Timer::Enabled { duration: 3600 })
+        .await
+        .unwrap();
+
+    // Send a saved message which will be deleted after 3600s
+    let now = time();
+    let msg = t.send_text(self_chat.id, "Message text").await;
+
+    check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3599, time() + 3601)
+        .await
+        .unwrap();
+
+    // Set DeleteDeviceAfter to 1800s. Then send a saved message which will
+    // still be deleted after 3600s because DeleteDeviceAfter doesn't apply to saved messages.
+    t.set_config(Config::DeleteDeviceAfter, Some("1800"))
+        .await?;
+
+    let now = time();
+    let msg = t.send_text(self_chat.id, "Message text").await;
+
+    check_msg_will_be_deleted(&t, msg.sender_msg_id, &self_chat, now + 3559, time() + 3601)
+        .await
+        .unwrap();
+
+    // Send a message to Bob which will be deleted after 1800s because of DeleteDeviceAfter.
+    let bob_chat = t.create_chat_with_contact("", "bob@example.net").await;
+    let now = time();
+    let msg = t.send_text(bob_chat.id, "Message text").await;
+
+    check_msg_will_be_deleted(
+        &t,
+        msg.sender_msg_id,
+        &bob_chat,
+        now + 1799,
+        // The message may appear to be sent MAX_SECONDS_TO_LEND_FROM_FUTURE later and
+        // therefore be deleted MAX_SECONDS_TO_LEND_FROM_FUTURE later.
+        time() + 1801 + MAX_SECONDS_TO_LEND_FROM_FUTURE,
+    )
+    .await
+    .unwrap();
+
+    // Enable ephemeral messages with Bob -> message will be deleted after 60s.
+    // This tests that the message is deleted at min(ephemeral deletion time, DeleteDeviceAfter deletion time).
+    bob_chat
+        .id
+        .set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
+        .await?;
+
+    let now = time();
+    let msg = t.send_text(bob_chat.id, "Message text").await;
+
+    check_msg_will_be_deleted(&t, msg.sender_msg_id, &bob_chat, now + 59, time() + 61)
+        .await
+        .unwrap();
+
+    Ok(())
+}
+
+async fn check_msg_will_be_deleted(
+    t: &TestContext,
+    msg_id: MsgId,
+    chat: &Chat,
+    not_deleted_at: i64,
+    deleted_at: i64,
+) -> Result<()> {
+    let next_expiration = next_expiration_timestamp(t).await.unwrap();
+
+    assert!(next_expiration > not_deleted_at);
+    delete_expired_messages(t, not_deleted_at).await?;
+
+    let loaded = Message::load_from_db(t, msg_id).await?;
+    assert!(!loaded.text.is_empty());
+    assert_eq!(loaded.chat_id, chat.id);
+
+    assert!(next_expiration < deleted_at);
+    delete_expired_messages(t, deleted_at).await?;
+    t.evtracker
+        .get_matching(|evt| {
+            if let EventType::MsgDeleted {
+                msg_id: event_msg_id,
+                ..
+            } = evt
+            {
+                *event_msg_id == msg_id
+            } else {
+                false
+            }
+        })
+        .await;
+
+    let loaded = Message::load_from_db_optional(t, msg_id).await?;
+    assert!(loaded.is_none());
+
+    // Check that the msg was deleted locally.
+    check_msg_is_deleted(t, chat, msg_id).await;
+
+    Ok(())
+}
+
+async fn check_msg_is_deleted(t: &TestContext, chat: &Chat, msg_id: MsgId) {
+    let chat_items = chat::get_chat_msgs(t, chat.id).await.unwrap();
+    // Check that the chat is empty except for possibly info messages:
+    for item in &chat_items {
+        if let ChatItem::Message { msg_id } = item {
+            let msg = Message::load_from_db(t, *msg_id).await.unwrap();
+            assert!(msg.is_info())
+        }
+    }
+
+    // Check that if there is a message left, the text and metadata are gone
+    if let Ok(msg) = Message::load_from_db(t, msg_id).await {
+        assert_eq!(msg.from_id, ContactId::UNDEFINED);
+        assert_eq!(msg.to_id, ContactId::UNDEFINED);
+        assert_eq!(msg.text, "");
+        let rawtxt: Option<String> = t
+            .sql
+            .query_get_value("SELECT txt_raw FROM msgs WHERE id=?;", (msg_id,))
+            .await
+            .unwrap();
+        assert!(rawtxt.is_none_or_empty(), "{rawtxt:?}");
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_delete_expired_imap_messages() -> Result<()> {
+    let t = TestContext::new_alice().await;
+    const HOUR: i64 = 60 * 60;
+    let now = time();
+    for (id, timestamp, ephemeral_timestamp) in &[
+        (900, now - 2 * HOUR, 0),
+        (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
+        (1010, now - 23 * HOUR, 0),
+        (1020, now - 21 * HOUR, 0),
+        (1030, now - 19 * HOUR, 0),
+        (2000, now - 18 * HOUR, now - HOUR),
+        (2020, now - 17 * HOUR, now + HOUR),
+        (3000, now + HOUR, 0),
+    ] {
+        let message_id = id.to_string();
+        t.sql
+               .execute(
+                   "INSERT INTO msgs (id, rfc724_mid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
+                   (id, &message_id, timestamp, ephemeral_timestamp),
+               )
+               .await?;
+        t.sql
+            .execute(
+                "INSERT INTO imap (rfc724_mid, folder, uid, target) VALUES (?,'INBOX',?, 'INBOX');",
+                (&message_id, id),
+            )
+            .await?;
+    }
+
+    async fn test_marked_for_deletion(context: &Context, id: u32) -> Result<()> {
+        assert_eq!(
+            context
+                .sql
+                .count(
+                    "SELECT COUNT(*) FROM imap WHERE target='' AND rfc724_mid=?",
+                    (id.to_string(),),
+                )
+                .await?,
+            1
+        );
+        Ok(())
+    }
+
+    async fn remove_uid(context: &Context, id: u32) -> Result<()> {
+        context
+            .sql
+            .execute("DELETE FROM imap WHERE rfc724_mid=?", (id.to_string(),))
+            .await?;
+        Ok(())
+    }
+
+    // This should mark message 2000 for deletion.
+    delete_expired_imap_messages(&t).await?;
+    test_marked_for_deletion(&t, 2000).await?;
+    remove_uid(&t, 2000).await?;
+    // No other messages are marked for deletion.
+    assert_eq!(
+        t.sql
+            .count("SELECT COUNT(*) FROM imap WHERE target=''", ())
+            .await?,
+        0
+    );
+
+    t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
+        .await?;
+    delete_expired_imap_messages(&t).await?;
+    test_marked_for_deletion(&t, 1000).await?;
+
+    MsgId::new(1000)
+        .update_download_state(&t, DownloadState::Available)
+        .await?;
+    t.sql
+        .execute("UPDATE imap SET target=folder WHERE rfc724_mid='1000'", ())
+        .await?;
+    delete_expired_imap_messages(&t).await?;
+    test_marked_for_deletion(&t, 1000).await?; // Delete downloadable anyway.
+    remove_uid(&t, 1000).await?;
+
+    t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
+        .await?;
+    delete_expired_imap_messages(&t).await?;
+    test_marked_for_deletion(&t, 1010).await?;
+    t.sql
+        .execute("UPDATE imap SET target=folder WHERE rfc724_mid='1010'", ())
+        .await?;
+
+    MsgId::new(1010)
+        .update_download_state(&t, DownloadState::Available)
+        .await?;
+    delete_expired_imap_messages(&t).await?;
+    // Keep downloadable for now.
+    assert_eq!(
+        t.sql
+            .count("SELECT COUNT(*) FROM imap WHERE target=''", ())
+            .await?,
+        0
+    );
+
+    t.set_config(Config::DeleteServerAfter, Some("1")).await?;
+    delete_expired_imap_messages(&t).await?;
+    test_marked_for_deletion(&t, 3000).await?;
+
+    Ok(())
+}
+
+// Regression test for a bug in the timer rollback protection.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_timer_references() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+
+    // Message with Message-ID <first@example.com> and no timer is received.
+    receive_imf(
+        &alice,
+        b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <first@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:10:00 +0000\n\
+                    \n\
+                    hello\n",
+        false,
+    )
+    .await?;
+
+    let msg = alice.get_last_msg().await;
+    let chat_id = msg.chat_id;
+    assert_eq!(chat_id.get_ephemeral_timer(&alice).await?, Timer::Disabled);
+
+    // Message with Message-ID <second@example.com> is received.
+    receive_imf(
+        &alice,
+        b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <second@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:11:00 +0000\n\
+                    Ephemeral-Timer: 60\n\
+                    \n\
+                    second message\n",
+        false,
+    )
+    .await?;
+    assert_eq!(
+        chat_id.get_ephemeral_timer(&alice).await?,
+        Timer::Enabled { duration: 60 }
+    );
+    let msg = alice.get_last_msg().await;
+
+    // Message is deleted when its timer expires.
+    msg.id.trash(&alice, false).await?;
+
+    // Message with Message-ID <third@example.com>, referencing <first@example.com> and
+    // <second@example.com>, is received.  The message <second@example.come> is not in the
+    // database anymore, so the timer should be applied unconditionally without rollback
+    // protection.
+    //
+    // Previously Delta Chat fallen back to using <first@example.com> in this case and
+    // compared received timer value to the timer value of the <first@example.com>. Because
+    // their timer values are the same ("disabled"), Delta Chat assumed that the timer was not
+    // changed explicitly and the change should be ignored.
+    //
+    // The message also contains a quote of the first message to test that only References:
+    // header and not In-Reply-To: is consulted by the rollback protection.
+    receive_imf(
+        &alice,
+        b"From: Bob <bob@example.com>\n\
+                    To: Alice <alice@example.org>\n\
+                    Chat-Version: 1.0\n\
+                    Subject: Subject\n\
+                    Message-ID: <third@example.com>\n\
+                    Date: Sun, 22 Mar 2020 00:12:00 +0000\n\
+                    References: <first@example.com> <second@example.com>\n\
+                    In-Reply-To: <first@example.com>\n\
+                    \n\
+                    > hello\n",
+        false,
+    )
+    .await?;
+
+    let msg = alice.get_last_msg().await;
+    assert_eq!(
+        msg.chat_id.get_ephemeral_timer(&alice).await?,
+        Timer::Disabled
+    );
+
+    Ok(())
+}
+
+// Tests that if we are offline for a time longer than the ephemeral timer duration, the message
+// is deleted from the chat but is still in the "smtp" table, i.e. will be sent upon a
+// successful reconnection.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_msg_offline() -> Result<()> {
+    let alice = TestContext::new_alice().await;
+    let chat = alice
+        .create_chat_with_contact("Bob", "bob@example.org")
+        .await;
+    let duration = 60;
+    chat.id
+        .set_ephemeral_timer(&alice, Timer::Enabled { duration })
+        .await?;
+    let mut msg = Message::new_text("hi".to_string());
+    assert!(chat::send_msg_sync(&alice, chat.id, &mut msg)
+        .await
+        .is_err());
+    let stmt = "SELECT COUNT(*) FROM smtp WHERE msg_id=?";
+    assert!(alice.sql.exists(stmt, (msg.id,)).await?);
+    let now = time();
+    check_msg_will_be_deleted(&alice, msg.id, &chat, now, now + i64::from(duration) + 1).await?;
+    assert!(alice.sql.exists(stmt, (msg.id,)).await?);
+
+    Ok(())
+}
+
+/// Tests that POI location is deleted when ephemeral message expires.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_ephemeral_poi_location() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let chat = alice.create_chat(bob).await;
+
+    let duration = 60;
+    chat.id
+        .set_ephemeral_timer(alice, Timer::Enabled { duration })
+        .await?;
+    let sent = alice.pop_sent_msg().await;
+    bob.recv_msg(&sent).await;
+
+    let mut poi_msg = Message::new_text("Here".to_string());
+    poi_msg.set_location(10.0, 20.0);
+
+    let alice_sent_message = alice.send_msg(chat.id, &mut poi_msg).await;
+    let bob_received_message = bob.recv_msg(&alice_sent_message).await;
+    markseen_msgs(bob, vec![bob_received_message.id]).await?;
+
+    for account in [alice, bob] {
+        let locations = location::get_range(account, None, None, 0, 0).await?;
+        assert_eq!(locations.len(), 1);
+    }
+
+    SystemTime::shift(Duration::from_secs(100));
+
+    for account in [alice, bob] {
+        delete_expired_messages(account, time()).await?;
+        let locations = location::get_range(account, None, None, 0, 0).await?;
+        assert_eq!(locations.len(), 0);
+    }
+
+    Ok(())
+}
+
+/// Tests that `.get_ephemeral_timer()` returns an error for invalid chat ID.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_get_ephemeral_timer_wrong_chat_id() -> Result<()> {
+    let context = TestContext::new().await;
+    let chat_id = ChatId::new(12345);
+    assert!(chat_id.get_ephemeral_timer(&context).await.is_err());
+
+    Ok(())
+}
+
+/// Tests that ephemeral timer is started when the chat is noticed.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_noticed_ephemeral_timer() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let chat = alice.create_chat(bob).await;
+    let duration = 60;
+    chat.id
+        .set_ephemeral_timer(alice, Timer::Enabled { duration })
+        .await?;
+    let bob_received_message = tcm.send_recv(alice, bob, "Hello!").await;
+
+    marknoticed_chat(bob, bob_received_message.chat_id).await?;
+    SystemTime::shift(Duration::from_secs(100));
+
+    delete_expired_messages(bob, time()).await?;
+
+    assert!(Message::load_from_db_optional(bob, bob_received_message.id)
+        .await?
+        .is_none());
+    Ok(())
+}
+
+/// Tests that archiving the chat starts ephemeral timer.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_archived_ephemeral_timer() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    let chat = alice.create_chat(bob).await;
+    let duration = 60;
+    chat.id
+        .set_ephemeral_timer(alice, Timer::Enabled { duration })
+        .await?;
+    let bob_received_message = tcm.send_recv(alice, bob, "Hello!").await;
+
+    bob_received_message
+        .chat_id
+        .set_visibility(bob, ChatVisibility::Archived)
+        .await?;
+    SystemTime::shift(Duration::from_secs(100));
+
+    delete_expired_messages(bob, time()).await?;
+
+    assert!(Message::load_from_db_optional(bob, bob_received_message.id)
+        .await?
+        .is_none());
+
+    // Bob mutes the chat so it is not unarchived.
+    set_muted(bob, bob_received_message.chat_id, MuteDuration::Forever).await?;
+
+    // Now test that for already archived chat
+    // timer is started if all archived chats are marked as noticed.
+    let bob_received_message_2 = tcm.send_recv(alice, bob, "Hello again!").await;
+    assert_eq!(bob_received_message_2.state, MessageState::InFresh);
+
+    marknoticed_chat(bob, DC_CHAT_ID_ARCHIVED_LINK).await?;
+    SystemTime::shift(Duration::from_secs(100));
+
+    delete_expired_messages(bob, time()).await?;
+
+    assert!(
+        Message::load_from_db_optional(bob, bob_received_message_2.id)
+            .await?
+            .is_none()
+    );
+
+    Ok(())
+}

src/events/payload.rs

@@ -59,7 +59,7 @@ pub enum EventType {
     /// or for functions that are expected to fail (eg. dc_continue_key_transfer())
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
-    /// in a messasge box then.
+    /// in a message box then.
     Error(String),
 
     /// An action cannot be performed because the user is not in the group.
@@ -97,6 +97,9 @@ pub enum EventType {
 
     /// Reactions for the message changed.
     IncomingReaction {
+        /// ID of the chat which the message belongs to.
+        chat_id: ChatId,
+
         /// ID of the contact whose reaction set is changed.
         contact_id: ContactId,
 
@@ -107,6 +110,24 @@ pub enum EventType {
         reaction: Reaction,
     },
 
+    /// A webxdc wants an info message or a changed summary to be notified.
+    IncomingWebxdcNotify {
+        /// ID of the chat.
+        chat_id: ChatId,
+
+        /// ID of the contact sending.
+        contact_id: ContactId,
+
+        /// ID of the added info message or webxdc instance in case of summary change.
+        msg_id: MsgId,
+
+        /// Text to notify.
+        text: String,
+
+        /// Link assigned to this notification, if any.
+        href: Option<String>,
+    },
+
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -332,6 +353,20 @@ pub enum EventType {
         chat_id: Option<ChatId>,
     },
 
+    /// Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+    ///
+    /// This event is only emitted by the account manager
+    AccountsChanged,
+
+    /// Inform that an account property that might be shown in the account list changed, namely:
+    /// - is_configured (see [crate::context::Context::is_configured])
+    /// - displayname
+    /// - selfavatar
+    /// - private_tag
+    ///
+    /// This event is emitted from the account whose property changed.
+    AccountsItemChanged,
+
     /// Event for using in tests, e.g. as a fence between normally generated events.
     #[cfg(test)]
     Test,

src/headerdef.rs

@@ -65,6 +65,15 @@ pub enum HeaderDef {
     ChatGroupMemberAdded,
     ChatContent,
 
+    /// Past members of the group.
+    ChatGroupPastMembers,
+
+    /// Space-separated timestamps of member addition
+    /// for members listed in the `To` field
+    /// followed by timestamps of member removal
+    /// for members listed in the `Chat-Group-Past-Members` field.
+    ChatGroupMemberTimestamps,
+
     /// Duration of the attached media file.
     ChatDuration,
 
@@ -73,6 +82,7 @@ pub enum HeaderDef {
 
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,
+    AutocryptGossip,
     AutocryptSetupMessage,
     SecureJoin,