ci: increase timeout in test_logged_ac_process_ffi_failure

This test keeps failing on macOS CI,
likely because it does not get enough resources
to finish in 10 seconds.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.82.0
+      RUSTUP_TOOLCHAIN: 1.83.0
     steps:
       - uses: actions/checkout@v4
         with:
@@ -37,8 +37,10 @@ jobs:
         run: cargo fmt --all -- --check
       - name: Run clippy
         run: scripts/clippy.sh
-      - name: Check
+      - name: Check with all features
         run: cargo check --workspace --all-targets --all-features
+      - name: Check with only default features
+        run: cargo check --all-targets
 
   npm_constants:
     name: Check if node constants are up to date
@@ -95,11 +97,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.82.0
+            rust: 1.83.0
           - os: windows-latest
-            rust: 1.82.0
+            rust: 1.83.0
           - os: macos-latest
-            rust: 1.82.0
+            rust: 1.83.0
 
           # Minimum Supported Rust Version = 1.77.0
           - os: ubuntu-latest
@@ -249,7 +251,7 @@ jobs:
 
       - name: Run python tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
@@ -314,6 +316,6 @@ jobs:
 
       - name: Run deltachat-rpc-client tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/jsonrpc.yml

@@ -33,7 +33,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/nix.yml

@@ -0,0 +1,104 @@
+name: Test Nix flake
+
+on:
+  pull_request:
+    paths:
+      - flake.nix
+      - flake.lock
+  push:
+    paths:
+      - flake.nix
+      - flake.lock
+    branches:
+      - main
+
+jobs:
+  format:
+    name: check flake formatting
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix fmt
+
+      # Check that formatting does not change anything.
+      - run: git diff --exit-code
+
+  build:
+    name: nix build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        installable:
+          # Ensure `nix develop` will work.
+          - devShells.x86_64-linux.default
+
+          - deltachat-python
+          - deltachat-repl
+          - deltachat-repl-aarch64-linux
+          - deltachat-repl-arm64-v8a-android
+          - deltachat-repl-armeabi-v7a-android
+          - deltachat-repl-armv6l-linux
+          - deltachat-repl-armv7l-linux
+          - deltachat-repl-i686-linux
+          - deltachat-repl-win32
+          - deltachat-repl-win64
+          - deltachat-repl-x86_64-linux
+          - deltachat-rpc-client
+          - deltachat-rpc-server
+          - deltachat-rpc-server-aarch64-linux
+          - deltachat-rpc-server-aarch64-linux-wheel
+          - deltachat-rpc-server-arm64-v8a-android
+          - deltachat-rpc-server-armeabi-v7a-android
+          - deltachat-rpc-server-armv6l-linux
+          - deltachat-rpc-server-armv6l-linux-wheel
+          - deltachat-rpc-server-armv7l-linux
+          - deltachat-rpc-server-armv7l-linux-wheel
+          - deltachat-rpc-server-i686-linux
+          - deltachat-rpc-server-i686-linux-wheel
+          - deltachat-rpc-server-source
+          - deltachat-rpc-server-win32
+          - deltachat-rpc-server-win32-wheel
+          - deltachat-rpc-server-win64
+          - deltachat-rpc-server-win64-wheel
+          - deltachat-rpc-server-x86_64-linux
+          - deltachat-rpc-server-x86_64-linux-wheel
+          - docs
+          - libdeltachat
+          - python-docs
+
+          # Fails to build
+          #- deltachat-repl-x86_64-android
+          #- deltachat-repl-x86-android
+          #- deltachat-rpc-server-x86_64-android
+          #- deltachat-rpc-server-x86-android
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build .#${{ matrix.installable }}
+
+  build-macos:
+    name: nix build on macOS
+    runs-on: macos-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        installable:
+          - deltachat-rpc-server-aarch64-darwin
+
+          # Fails to bulid
+          # - deltachat-rpc-server-x86_64-darwin
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build .#${{ matrix.installable }}

.github/workflows/node-tests.yml

@@ -64,5 +64,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          CHATMAIL_DOMAIN: ${{ vars.CHATMAIL_DOMAIN }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

CHANGELOG.md

@@ -1,5 +1,522 @@
 # Changelog
 
+## [1.152.2] - 2024-12-24
+
+### Features / Changes
+
+- Emit ImexProgress(1) after receiving backup size.
+- `delete_msgs`: Use `transaction()` instead of `call_write()`.
+- Start ephemeral timers when the chat is noticed.
+- Start ephemeral timers when the chat is archived.
+- Revalidate HTTP cache entries once per minute maximum.
+
+### Fixes
+
+- Reduce number of `repeat_vars()` calls.
+- `sanitise_name`: Don't consider punctuation and control chars as part of file extension ([#6362](https://github.com/deltachat/deltachat-core-rust/pull/6362)).
+
+### Refactor
+
+- Remove marknoticed_chat_if_older_than().
+
+### Miscellaneous Tasks
+
+- Remove contrib/ directory.
+
+## [1.152.1] - 2024-12-17
+
+### Build system
+
+- Downgrade Rust version used to build binaries.
+- Reduce MSRV to 1.77.0.
+
+## [1.152.0] - 2024-12-12
+
+### API-Changes
+
+- [**breaking**] Remove `dc_prepare_msg` and `dc_msg_is_increation`.
+
+### Build system
+
+- Increase MSRV to 1.81.0.
+
+### Features / Changes
+
+- Cache HTTP GET requests.
+- Prefix server-url in info.
+- Set `mime_modified` for the last message part, not the first ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)).
+
+### Fixes
+
+- Render "message" parts in multipart messages' HTML ([#4462](https://github.com/deltachat/deltachat-core-rust/pull/4462)).
+- Ignore garbage at the end of the keys.
+
+## [1.151.6] - 2024-12-11
+
+### Features / Changes
+
+- Don't add "Failed to send message to ..." info messages to group chats.
+- Add info messages about implicit membership changes if group member list is recreated ([#6314](https://github.com/deltachat/deltachat-core-rust/pull/6314)).
+
+### Fixes
+
+- Add self-addition message to chat when recreating member list.
+- Do not subscribe to heartbeat if already subscribed via metadata.
+
+### Build system
+
+- Add idna 0.5.0 exception into deny.toml.
+
+### Documentation
+
+- Update links to Node.js bindings in the README.
+
+### Refactor
+
+- Factor out `wait_for_all_work_done()`.
+
+### Tests
+
+- Notifiy more prominently & in more tests about false positives when running `cargo test` ([#6308](https://github.com/deltachat/deltachat-core-rust/pull/6308)).
+
+## [1.151.5] - 2024-12-05
+
+### API-Changes
+
+- [**breaking**] Remove dc_all_work_done().
+
+### Security
+
+- cargo: Update rPGP to 0.14.2.
+
+  This fixes [Panics on Malformed Untrusted Input](https://github.com/rpgp/rpgp/security/advisories/GHSA-9rmp-2568-59rv)
+  and [Potential Resource Exhaustion when handling Untrusted Messages](https://github.com/rpgp/rpgp/security/advisories/GHSA-4grw-m28r-q285).
+  This allows the attacker to crash the application via specially crafted messages and keys.
+  We recommend all users and bot operators to upgrade to the latest version.
+  There is no impact on the confidentiality of the messages and keys so no action other than upgrading is needed.
+
+### Fixes
+
+- Store plaintext in mime_headers of truncated sent messages ([#6273](https://github.com/deltachat/deltachat-core-rust/pull/6273)).
+
+### Documentation
+
+- Document `push` module.
+- Remove mention of non-existent `nightly` feature.
+
+### Tests
+
+- Fix panic in `receive_emails` benchmark ([#6306](https://github.com/deltachat/deltachat-core-rust/pull/6306)).
+
+## [1.151.4] - 2024-12-03
+
+### Features / Changes
+
+- Encrypt notification tokens.
+
+### Fixes
+
+- Replace connectivity state "Connected" with "Preparing".
+
+### Miscellaneous Tasks
+
+- Beta clippy suggestions ([#6271](https://github.com/deltachat/deltachat-core-rust/pull/6271)).
+
+### Tests
+
+- Fix `cargo check` for `receive_emails` benchmark.
+
+### CI
+
+- Also run cargo check without all-features.
+
+## [1.151.3] - 2024-12-02
+
+### API-Changes
+
+- Remove experimental `request_internet_access` option from webxdc's `manifest.toml`.
+- Add getWebxdcHref to json api ([#6281](https://github.com/deltachat/deltachat-core-rust/pull/6281)).
+
+### CI
+
+- Update Rust to 1.83.0.
+
+### Documentation
+
+- Update dc_msg_get_info_type() and dc_get_securejoin_qr() ([#6269](https://github.com/deltachat/deltachat-core-rust/pull/6269)).
+- Fix references to iroh-related headers in peer_channels docs.
+- Improve CFFI docs, link to corresponding JSON-RPC docs.
+
+### Features / Changes
+
+- Allow the user to replace maps integration ([#5678](https://github.com/deltachat/deltachat-core-rust/pull/5678)).
+- Mark saved messages chat as protected.
+
+### Fixes
+
+- Close iroh endpoint when I/O is stopped.
+- Do not add protection messages to Saved Messages chat.
+- Mark Saved Messages chat as protected if it exists.
+- Sync chat action even if sync message arrives before first one from contact ([#6259](https://github.com/deltachat/deltachat-core-rust/pull/6259)).
+
+### Refactor
+
+- Remove some .unwrap() calls.
+- Create_status_update_record: Remove double check of info_msg_id.
+- Use Option::or_else() to dedup emitting IncomingWebxdcNotify.
+
+## [1.151.2] - 2024-11-26
+
+### API-Changes
+
+- Deprecate webxdc `descr` parameter ([#6255](https://github.com/deltachat/deltachat-core-rust/pull/6255)).
+
+### Features / Changes
+
+- AEAP: Check that the old peerstate verified key fingerprint hasn't changed when removing it.
+- Add `AccountsChanged` and `AccountsItemChanged` events ([#6118](https://github.com/deltachat/deltachat-core-rust/pull/6118)).
+- Do not use format=flowed in outgoing messages ([#6256](https://github.com/deltachat/deltachat-core-rust/pull/6256)).
+- Add webxdc limits api.
+- Add href to IncomingWebxdcNotify event ([#6266](https://github.com/deltachat/deltachat-core-rust/pull/6266)).
+
+### Fixes
+
+- Revert treating some transient SMTP errors as permanent.
+
+### Refactor
+
+- Create_status_update_record: Get rid of `notify` var.
+
+### Tests
+
+- Check that IncomingMsg isn't emitted for reactions.
+
+## [1.151.1] - 2024-11-24
+
+### Build system
+
+- nix: Fix deltachat-rpc-server-source installable.
+
+### CI
+
+- Test building nix targets to avoid regressions.
+
+## [1.151.0] - 2024-11-23
+
+### Features / Changes
+
+- Trim whitespace from scanned QR codes.
+- Use privacy-preserving webxdc addresses ([#6237](https://github.com/deltachat/deltachat-core-rust/pull/6237)).
+- Webxdc notify ([#6230](https://github.com/deltachat/deltachat-core-rust/pull/6230)).
+- `update.href` api ([#6248](https://github.com/deltachat/deltachat-core-rust/pull/6248)).
+
+### Fixes
+
+- Never notify SELF ([#6251](https://github.com/deltachat/deltachat-core-rust/pull/6251)).
+
+### Build system
+
+- Use underscores in deltachat-rpc-server source package filename.
+- Remove imap_tools from dependencies ([#6238](https://github.com/deltachat/deltachat-core-rust/pull/6238)).
+- cargo: Update Rustls from 0.23.14 to 0.23.18.
+- deps: Bump curve25519-dalek from 3.2.0 to 4.1.3 in /fuzz.
+
+### Documentation
+
+- Move style guide into a separate document.
+- Clarify DC_EVENT_INCOMING_WEBXDC_NOTIFY documentation ([#6249](https://github.com/deltachat/deltachat-core-rust/pull/6249)).
+
+### Tests
+
+- After AEAP, 1:1 chat isn't available for sending, but unprotected groups are ([#6222](https://github.com/deltachat/deltachat-core-rust/pull/6222)).
+
+## [1.150.0] - 2024-11-21
+
+### API-Changes
+
+- Correct `DC_CERTCK_ACCEPT_*` values and docs ([#6176](https://github.com/deltachat/deltachat-core-rust/pull/6176)).
+
+### Features / Changes
+
+- Use Rustls for connections with strict TLS ([#6186](https://github.com/deltachat/deltachat-core-rust/pull/6186)).
+- Experimental header protection for Autocrypt.
+- Tune down io-not-started info in connectivity-html.
+- Clear config cache in start_io() ([#6228](https://github.com/deltachat/deltachat-core-rust/pull/6228)).
+- Line-before-quote may be up to 120 character long instead of 80.
+- Use i.delta.chat in qr codes ([#6223](https://github.com/deltachat/deltachat-core-rust/pull/6223)).
+
+### Fixes
+
+- Prevent accidental wrong-password-notifications ([#6122](https://github.com/deltachat/deltachat-core-rust/pull/6122)).
+- Remove footers from "Show Full Message...".
+- `send_msg_to_smtp`: Return Ok if `smtp` row is deleted in parallel.
+- Only add "member added/removed" messages if they actually do that ([#5992](https://github.com/deltachat/deltachat-core-rust/pull/5992)).
+- Do not fail to load chatlist summary if the message got removed.
+- deltachat-jsonrpc: Do not fail `get_chatlist_items_by_entries` if the message got deleted.
+- deltachat-jsonrpc: Do not fail `get_draft` if draft is deleted.
+- `markseen_msgs`: Limit not yet downloaded messages state to `InNoticed` ([#2970](https://github.com/deltachat/deltachat-core-rust/pull/2970)).
+- Update state of message when fully downloading it.
+- Dont overwrite equal drafts ([#6212](https://github.com/deltachat/deltachat-core-rust/pull/6212)).
+
+### Build system
+
+- Silence RUSTSEC-2024-0384.
+- cargo: Update rPGP from 0.13.2 to 0.14.0.
+- cargo: Update futures-concurrency from 7.6.1 to 7.6.2.
+- Update flake.nix ([#6200](https://github.com/deltachat/deltachat-core-rust/pull/6200))
+
+### CI
+
+- Ensure flake is formatted.
+
+### Documentation
+
+- Scanned proxies are added and normalized.
+
+### Refactor
+
+- Fix nightly clippy warnings.
+- Remove slicing from `is_file_in_use`.
+- Remove unnecessary `allow(clippy::indexing_slicing)`.
+- Don't use slicing in `remove_nonstandard_footer`.
+- Do not use slicing in `qr` module.
+- Eliminate indexing in `compute_mailinglist_name`.
+- Remove unused `allow(clippy::indexing_slicing)`.
+- Remove indexing/slicing from `remove_message_footer`.
+- Remove indexing/slicing from `squash_attachment_parts`.
+- Remove unused allow(clippy::indexing_slicing) for heuristically_parse_ndn.
+- Remove indexing/slicing from `parse_message_ids`.
+- Remove slicing from `remove_bottom_quote`.
+- Get rid of slicing in `remove_top_quote`.
+- Remove unused allow(clippy::indexing_slicing) from 'truncate'.
+- Forbid clippy::indexing_slicing.
+- Forbid clippy::string_slice.
+- Delete chat in a transaction.
+- Fix typo in `context.rs`.
+
+### Tests
+
+- Remove all calls to print() from deltachat-rpc-client tests.
+- Reply to protected group from MUA.
+- Mark not downloaded message as seen ([#2970](https://github.com/deltachat/deltachat-core-rust/pull/2970)).
+- Mark `receive_imf()` as only for tests and "internals" feature ([#6235](https://github.com/deltachat/deltachat-core-rust/pull/6235)).
+
+## [1.149.0] - 2024-11-05
+
+### Build system
+
+- Update tokio to 1.41 and Android NDK to r27.
+- `nix flake update android`.
+
+### Fixes
+
+- cargo: Update iroh to 0.28.1.
+  This fixes the problem with iroh not sending the `Host:` header and not being able to connect to relays behind nginx reverse proxy.
+
+## [1.148.7] - 2024-11-03
+
+### API-Changes
+
+- Add API to reset contact encryption.
+
+### Features / Changes
+
+- Emit chatlist events only if message still exists.
+
+### Fixes
+
+- send_msg_to_smtp: Do not fail if the message does not exist anymore.
+- Do not percent-encode dot when passing to autoconfig server.
+- Save contact name from SecureJoin QR to `authname`, not to `name` ([#6115](https://github.com/deltachat/deltachat-core-rust/pull/6115)).
+- Always exit fake IDLE after at most 60 seconds.
+- Concat NDNs ([#6129](https://github.com/deltachat/deltachat-core-rust/pull/6129)).
+
+### Refactor
+
+- Remove `has_decrypted_pgp_armor()`.
+
+### Miscellaneous Tasks
+
+- Update dependencies.
+
+## [1.148.6] - 2024-10-31
+
+### API-Changes
+
+- Add Message::new_text() ([#6123](https://github.com/deltachat/deltachat-core-rust/pull/6123)).
+- Add `MessageSearchResult.chat_id` ([#6120](https://github.com/deltachat/deltachat-core-rust/pull/6120)).
+
+### Features / Changes
+
+- Enable Webxdc realtime by default ([#6125](https://github.com/deltachat/deltachat-core-rust/pull/6125)).
+
+### Fixes
+
+- Save full text to mime_headers for long outgoing messages ([#6091](https://github.com/deltachat/deltachat-core-rust/pull/6091)).
+- Show root SMTP connection failure in connectivity view ([#6121](https://github.com/deltachat/deltachat-core-rust/pull/6121)).
+- Skip IDLE if we got unsolicited FETCH ([#6130](https://github.com/deltachat/deltachat-core-rust/pull/6130)).
+
+### Miscellaneous Tasks
+
+- Silence another rust-analyzer false-positive ([#6124](https://github.com/deltachat/deltachat-core-rust/pull/6124)).
+- cargo: Upgrade iroh to 0.26.0.
+
+### Refactor
+
+- Directly use connectives ([#6128](https://github.com/deltachat/deltachat-core-rust/pull/6128)).
+- Use Message::new_text() more ([#6127](https://github.com/deltachat/deltachat-core-rust/pull/6127)).
+
+## [1.148.5] - 2024-10-27
+
+### Fixes
+
+- Set Config::NotifyAboutWrongPw before saving configuration ([#5896](https://github.com/deltachat/deltachat-core-rust/pull/5896)).
+- Do not take write lock for maybe_network_lost() and set_push_device_token().
+- Do not lock the account manager for the whole duration of background_fetch.
+
+### Features / Changes
+
+- Auto-restore 1:1 chat protection after receiving old unverified message.
+
+### CI
+
+- Take `CHATMAIL_DOMAIN` from variables instead of secrets.
+
+### Other
+
+- Revert "build: nix flake update fenix" to fix `nix build .#deltachat-rpc-server-armeabi-v7a-android`.
+
+### Refactor
+
+- Receive_imf::add_parts: Remove excessive `from_id == ContactId::SELF` checks.
+- Factor out `add_gossip_peer_from_header()`.
+
+## [1.148.4] - 2024-10-24
+
+### Features / Changes
+
+- Jsonrpc: add `private_tag` to `Account::Configured` Object ([#6107](https://github.com/deltachat/deltachat-core-rust/pull/6107)).
+
+### Fixes
+
+- Normalize proxy URLs before saving into proxy_url.
+- Do not wait for connections in maybe_add_gossip_peers().
+
+## [1.148.3] - 2024-10-24
+
+### Fixes
+
+- Fix reception of realtime advertisements.
+
+### Features / Changes
+
+- Allow sending realtime messages up to 128 KB in size.
+
+### API-Changes
+
+- deltachat-rpc-client: Add EventType.WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED.
+
+### Documentation
+
+- Fix DC_QR_PROXY docs ([#6099](https://github.com/deltachat/deltachat-core-rust/pull/6099)).
+
+### Refactor
+
+- Generate topic inside create_iroh_header().
+
+### Tests
+
+- Test that realtime advertisements work after chatting.
+
+## [1.148.2] - 2024-10-23
+
+### Fixes
+
+- Never initialize Iroh if realtime is disabled.
+
+### Features / Changes
+
+- Add more logging for iroh initialization and peer addition.
+
+### Build system
+
+- `nix flake update nixpkgs`.
+- `nix flake update fenix`.
+
+## [1.148.1] - 2024-10-23
+
+### Build system
+
+- Revert "build: nix flake update"
+
+This reverts commit 6f22ce2722b51773d7fbb0d89e4764f963cafd91..
+
+## [1.148.0] - 2024-10-22
+
+### API-Changes
+
+- Create QR codes from any data ([#6090](https://github.com/deltachat/deltachat-core-rust/pull/6090)).
+- Add delta chat logo to QR codes ([#6093](https://github.com/deltachat/deltachat-core-rust/pull/6093)).
+- Add realtime advertisement received event ([#6043](https://github.com/deltachat/deltachat-core-rust/pull/6043)).
+- Notify adding reactions ([#6072](https://github.com/deltachat/deltachat-core-rust/pull/6072))
+- Internal profile names ([#6088](https://github.com/deltachat/deltachat-core-rust/pull/6088)).
+
+### Features / Changes
+
+- IMAP COMPRESS support.
+- Sort received outgoing message down if it's fresher than all non fresh messages.
+- Prioritize cached results if DNS resolver returns many results.
+- Add in-memory cache for DNS.
+- deltachat-repl: Built-in QR code printer.
+- Log the logic for (not) doing AEAP.
+- Log when late Autocrypt header is ignored.
+- Add more context to `send_msg` errors.
+
+### Fixes
+
+- Replace old draft with a new one atomically.
+- ChatId::maybe_delete_draft: Don't delete message if it's not a draft anymore ([#6053](https://github.com/deltachat/deltachat-core-rust/pull/6053)).
+- Call update_connection_history for proxified connections.
+- sql: Set PRAGMA query_only to avoid writing on read-only connections.
+- sql: Run `PRAGMA incremental_vacuum` on a write connection.
+- Increase MAX_SECONDS_TO_LEND_FROM_FUTURE to 30.
+
+### Build system
+
+- Nix flake update.
+- Resolve warning about default-features, and make it possible to disable vendoring ([#6079](https://github.com/deltachat/deltachat-core-rust/pull/6079)).
+- Silence a rust-analyzer false-positive ([#6077](https://github.com/deltachat/deltachat-core-rust/pull/6077)).
+
+### CI
+
+- Update Rust to 1.82.0.
+
+### Documentation
+
+- Set_protection_for_timestamp_sort does not send messages.
+- Document MimeFactory.req_mdn.
+- Fix `too_long_first_doc_paragraph` clippy lint.
+
+### Refactor
+
+- Update_msg_state: Don't avoid downgrading OutMdnRcvd to OutDelivered.
+- Fix elided_named_lifetimes warning.
+- set_protection_for_timestamp_sort: Do not log bubbled up errors.
+- Fix clippy::needless_lifetimes warnings.
+- Use `HeaderDef` constant for Chat-Disposition-Notification-To.
+- Resultify get_self_fingerprint().
+- sql: Move write mutex into connection pool.
+
+### Tests
+
+- test_qr_setup_contact_svg: Stop testing for no display name.
+- Always gossip if gossip_period is set to 0.
+- test_aeap_flow_verified: Wait for "member added" before sending messages ([#6057](https://github.com/deltachat/deltachat-core-rust/pull/6057)).
+- Make test_verified_group_member_added_recovery more reliable.
+- test_aeap_flow_verified: Do not start ac1new.
+- Fix `test_securejoin_after_contact_resetup` flakiness.
+- Message from old setup preserves contact verification, but breaks 1:1 protection.
+
 ## [1.147.1] - 2024-10-13
 
 ### Build system
@@ -20,7 +537,7 @@
 - Reset quota on configured address change ([#5908](https://github.com/deltachat/deltachat-core-rust/pull/5908)).
 - Do not emit progress 1000 when configuration is cancelled.
 - Assume file extensions are 32 chars max and don't contain whitespace ([#5338](https://github.com/deltachat/deltachat-core-rust/pull/5338)).
-- Readd tokens.foreign_id column ([#6038](https://github.com/deltachat/deltachat-core-rust/pull/6038)).
+- Re-add tokens.foreign_id column ([#6038](https://github.com/deltachat/deltachat-core-rust/pull/6038)).
 
 ### Miscellaneous Tasks
 
@@ -611,7 +1128,7 @@
 
 ### Tests
 
-- deltachat-rpc-client: reenable `log_cli`.
+- deltachat-rpc-client: re-enable `log_cli`.
 
 ## [1.140.0] - 2024-06-04
 
@@ -1548,7 +2065,7 @@
   - Mark 1:1 chat as verified for Bob early. 1:1 chat with Alice is verified as soon as Alice's key is verified rather than at the end of the protocol.
 - Put Message-ID into hidden headers and take it from there on receiver ([#4798](https://github.com/deltachat/deltachat-core-rust/pull/4798)). This works around servers which generate their own Message-ID and overwrite the one generated by Delta Chat.
 - deltachat-repl: Enable INFO logging by default and add timestamps.
-- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elments based on the configuration key which is a part of the event.
+- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elements based on the configuration key which is a part of the event.
 - Sync contact creation/rename across devices ([#5163](https://github.com/deltachat/deltachat-core-rust/pull/5163)).
 - Encrypt MDNs ([#5175](https://github.com/deltachat/deltachat-core-rust/pull/5175)).
 - Only try to configure non-strict TLS checks if explicitly set ([#5181](https://github.com/deltachat/deltachat-core-rust/pull/5181)).
@@ -4265,14 +4782,10 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 - new qr-code type `DC_QR_WEBRTC` #1779
 
-- new `dc_chatlist_get_summary2()` api #1771
-
 - tweak smtp-timeout for larger mails #1782
 
 - optimize read-receipts #1765
 
-- Allow http scheme for DCACCOUNT URLs #1770
-
 - improve tests #1769
 
 - bug fixes #1766 #1772 #1773 #1775 #1776 #1777
@@ -5008,3 +5521,23 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.146.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.145.0..v1.146.0
 [1.147.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.146.0..v1.147.0
 [1.147.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.147.0..v1.147.1
+[1.148.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.147.1..v1.148.0
+[1.148.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.0..v1.148.1
+[1.148.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.1..v1.148.2
+[1.148.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.2..v1.148.3
+[1.148.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.3..v1.148.4
+[1.148.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.4..v1.148.5
+[1.148.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.5..v1.148.6
+[1.148.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.6..v1.148.7
+[1.149.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.148.7..v1.149.0
+[1.150.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.149.0..v1.150.0
+[1.151.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.150.0..v1.151.0
+[1.151.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.0..v1.151.1
+[1.151.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.1..v1.151.2
+[1.151.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.2..v1.151.3
+[1.151.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.3..v1.151.4
+[1.151.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.4..v1.151.5
+[1.151.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.5..v1.151.6
+[1.152.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.151.6..v1.152.0
+[1.152.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.152.0..v1.152.1
+[1.152.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.152.1..v1.152.2

CONTRIBUTING.md

@@ -1,6 +1,6 @@
-# Contributing guidelines
+# Contributing to Delta Chat
 
-## Reporting bugs
+## Bug reports
 
 If you found a bug, [report it on GitHub](https://github.com/deltachat/deltachat-core-rust/issues).
 If the bug you found is specific to
@@ -9,13 +9,27 @@ If the bug you found is specific to
 [Desktop](https://github.com/deltachat/deltachat-desktop/issues),
 report it to the corresponding repository.
 
-## Proposing features
+## Feature proposals
 
 If you have a feature request, create a new topic on the [forum](https://support.delta.chat/).
 
-## Contributing code
+## Code contributions
 
-If you want to contribute a code, [open a Pull Request](https://github.com/deltachat/deltachat-core-rust/pulls).
+If you want to contribute a code, follow this guide.
+
+1. **Select an issue to work on.**
+
+   If you have an write access to the repository, assign the issue to yourself.
+   Otherwise state in the comment that you are going to work on the issue
+   to avoid duplicate work.
+
+   If the issue does not exist yet, create it first.
+
+2. **Write the code.**
+
+   Follow the [coding conventions](STYLE.md) when writing the code.
+
+3. **Commit the code.**
 
    If you have write access to the repository,
    push a branch named `<username>/<feature>`
@@ -23,75 +37,6 @@ so it is clear who is responsible for the branch,
    and open a PR proposing to merge the change.
    Otherwise fork the repository and create a branch in your fork.
 
-You can find the list of good first issues
-and a link to this guide
-on the contributing page: <https://github.com/deltachat/deltachat-core-rust/contribute>
-
-### Coding conventions
-
-We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
-Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
-
-### SQL
-
-Multi-line SQL statements should be formatted using string literals,
-for example
-```
-    sql.execute(
-        "CREATE TABLE messages (
-id INTEGER PRIMARY KEY AUTOINCREMENT,
-text TEXT DEFAULT '' NOT NULL -- message text
-) STRICT",
-    )
-    .await?;
-```
-
-Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
-or [`indoc!](https://docs.rs/indoc).
-Do not escape newlines like this:
-```
-    sql.execute(
-        "CREATE TABLE messages ( \
-id INTEGER PRIMARY KEY AUTOINCREMENT, \
-text TEXT DEFAULT '' NOT NULL \
-) STRICT",
-    )
-    .await?;
-```
-Escaping newlines
-is prone to errors like this if space before backslash is missing:
-```
-"SELECT foo\
- FROM bar"
-```
-Literal above results in `SELECT fooFROM bar` string.
-This style also does not allow using `--` comments.
-
----
-
-Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
-to make SQLite check column types.
-
-Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
-This avoids reuse of the row IDs and can avoid dangerous bugs
-like forwarding wrong message because the message was deleted
-and another message took its row ID.
-
-Declare all new columns as `NOT NULL`
-and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
-Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
-If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
-Use `HAVING COUNT(*) > 0` clause
-to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
-
-Don't delete unused columns too early, but maybe after several months/releases, unused columns are
-still used by older versions, so deleting them breaks downgrading the core or importing a backup in
-an older version. Also don't change the column type, consider adding a new column with another name
-instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
-keyword doesn't help here.
-
-### Commit messages
-
    Commit messages follow the [Conventional Commits] notation.
    We use [git-cliff] to generate the changelog from commit messages before the release.
 
@@ -109,14 +54,8 @@ The following prefix types are used:
    - `docs`: Documentation changes, e.g. "docs: add contributing guidelines"
    - `chore`: miscellaneous tasks, e.g. "chore: add `.DS_Store` to `.gitignore`"
 
-Release preparation commits are marked as "chore(release): prepare for vX.Y.Z".
+   Release preparation commits are marked as "chore(release): prepare for X.Y.Z"
-
+   as described in [releasing guide](RELEASE.md).
-If you intend to squash merge the PR from the web interface,
-make sure the PR title follows the conventional commits notation
-as it will end up being a commit title.
-Otherwise make sure each commit title follows the conventional commit notation.
-
-#### Breaking Changes
 
    Use a `!` to mark breaking changes, e.g. "api!: Remove `dc_chat_can_send`".
 
@@ -128,59 +67,56 @@ fix: Fix race condition and db corruption when a message was received during bac
    BREAKING CHANGE: You have to call `dc_stop_io()`/`dc_start_io()` before/after `dc_imex(DC_IMEX_EXPORT_BACKUP)`
    ```
 
-#### Multiple Changes in one PR
+4. [**Open a Pull Request**](https://github.com/deltachat/deltachat-core-rust/pulls).
 
-If you have multiple changes in one PR, create multiple conventional commits, and then do a rebase merge. Otherwise, you should usually do a squash merge.
+   Refer to the corresponding issue.
 
-[Clippy]: https://doc.rust-lang.org/clippy/
+   If you intend to squash merge the PR from the web interface,
-[Conventional Commits]: https://www.conventionalcommits.org/
+   make sure the PR title follows the conventional commits notation
-[git-cliff]: https://git-cliff.org/
+   as it will end up being a commit title.
+   Otherwise make sure each commit title follows the conventional commit notation.
 
-### Errors
+5. **Make sure all CI checks succeed.**
 
-Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.
+   CI runs the tests and checks code formatting.
-When using [`Context`](https://docs.rs/anyhow/latest/anyhow/trait.Context.html),
-capitalize it but do not add a full stop as the contexts will be separated by `:`.
-For example:
-```
-.with_context(|| format!("Unable to trash message {msg_id}"))
-```
 
-All errors should be handled in one of these ways:
+   While it is running, self-review your PR to make sure all the changes you expect are there
-- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+   and there are no accidentally committed unrelated changes and files.
-- With `.log_err().ok()`.
-- Bubbled up with `?`.
 
-`backtrace` feature is enabled for `anyhow` crate
+   Push the necessary fixup commits or force-push to your branch if needed.
-and `debug = 1` option is set in the test profile.
-This allows to run `RUST_BACKTRACE=1 cargo test`
-and get a backtrace with line numbers in resultified tests
-which return `anyhow::Result`.
 
-### Logging
+6. **Ask for review.**
 
-For logging, use `info!`, `warn!` and `error!` macros.
+   Use built-in GitHub feature to request a review from suggested reviewers.
-Log messages should be capitalized and have a full stop in the end. For example:
-```
-info!(context, "Ignoring addition of {added_addr:?} to {chat_id}.");
-```
 
-Format anyhow errors with `{:#}` to print all the contexts like this:
+   If you do not have write access to the repository, ask for review in the comments.
-```
-error!(context, "Failed to set selfavatar timestamp: {err:#}.");
-```
 
-### Reviewing
+7. **Merge the PR.**
 
    Once a PR has an approval and passes CI, it can be merged.
 
-PRs from a branch created in the main repository, i.e. authored by those who have write access, are merged by their authors.
+   PRs from a branch created in the main repository,
+   i.e. authored by those who have write access, are merged by their authors.
+
    This is to ensure that PRs are merged as intended by the author,
    e.g. as a squash merge, by rebasing from the web interface or manually from the command line.
 
+   If you have multiple changes in one PR, do a rebase merge.
+   Otherwise, you should usually do a squash merge.
+
+   If PR author does not have write access to the repository,
+   maintainers who reviewed the PR can merge it.
+
    If you do not have access to the repository and created a PR from a fork,
    ask the maintainers to merge the PR and say how it should be merged.
 
 ## Other ways to contribute
 
 For other ways to contribute, refer to the [website](https://delta.chat/en/contribute).
+
+You can find the list of good first issues
+and a link to this guide
+on the contributing page: <https://github.com/deltachat/deltachat-core-rust/contribute>
+
+[Conventional Commits]: https://www.conventionalcommits.org/
+[git-cliff]: https://git-cliff.org/

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.147.1"
+version = "1.152.2"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.77"
@@ -39,20 +39,20 @@ format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = { workspace = true }
-async-broadcast = "0.7.1"
+async-broadcast = "0.7.2"
 async-channel = { workspace = true }
 async-imap = { version = "0.10.2", default-features = false, features = ["runtime-tokio", "compress"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
-async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
+async-smtp = { version = "0.10", default-features = false, features = ["runtime-tokio"] }
 async_zip = { version = "0.0.17", default-features = false, features = ["deflate", "tokio-fs"] }
 base64 = { workspace = true }
-brotli = { version = "6", default-features=false, features = ["std"] }
+brotli = { version = "7", default-features=false, features = ["std"] }
 bytes = "1"
 chrono = { workspace = true, features = ["alloc", "clock", "std"] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
-fast-socks5 = "0.9"
+fast-socks5 = "0.10"
 fd-lock = "4"
 futures-lite = { workspace = true }
 futures = { workspace = true }
@@ -61,11 +61,11 @@ hickory-resolver = "=0.25.0-alpha.2"
 http-body-util = "0.1.2"
 humansize = "2"
 hyper = "1"
-hyper-util = "0.1.9"
+hyper-util = "0.1.10"
-image = { version = "0.25.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.25.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-iroh-gossip = { version = "0.25.0", default-features = false, features = ["net"] }
+iroh-gossip = { version = "0.28.1", default-features = false, features = ["net"] }
-iroh-net = { version = "0.25.0", default-features = false }
+iroh-net = { version = "0.28.1", default-features = false }
-kamadak-exif = "0.5.3"
+kamadak-exif = "0.6.1"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = { workspace = true }
 mailparse = "0.15"
@@ -76,22 +76,23 @@ num-traits = { workspace = true }
 once_cell = { workspace = true }
 parking_lot = "0.12"
 percent-encoding = "2.3"
-pgp = { version = "0.13.2", default-features = false }
+pgp = { version = "0.14.2", default-features = false }
 pin-project = "1"
 qrcodegen = "1.7.0"
-quick-xml = "0.36"
+quick-xml = "0.37"
 quoted_printable = "0.5"
 rand = { workspace = true }
 regex = { workspace = true }
 rusqlite = { workspace = true, features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustls-pki-types = "1.9.0"
+rustls-pki-types = "1.10.1"
-rustls = { version = "0.23.13", default-features = false }
+rustls = { version = "0.23.19", default-features = false }
 sanitize-filename = { workspace = true }
 serde_json = { workspace = true }
 serde_urlencoded = "0.7.1"
 serde = { workspace = true, features = ["derive"] }
 sha-1 = "0.10"
+sha2 = "0.10"
 shadowsocks = { version = "1.21.0", default-features = false, features = ["aead-cipher-2022"] }
 smallvec = "1.13.2"
 strum = "0.26"
@@ -100,15 +101,15 @@ tagger = "4.3.4"
 textwrap = "0.16.1"
 thiserror = { workspace = true }
 tokio-io-timeout = "1.2.0"
-tokio-rustls = { version = "0.26.0", default-features = false }
+tokio-rustls = { version = "0.26.1", default-features = false }
-tokio-stream = { version = "0.1.16", features = ["fs"] }
+tokio-stream = { version = "0.1.17", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = { workspace = true }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 toml = "0.8"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
-webpki-roots = "0.26.6"
+webpki-roots = "0.26.7"
 
 [dev-dependencies]
 anyhow = { workspace = true, features = ["backtrace"] } # Enable `backtrace` feature in tests.
@@ -149,6 +150,7 @@ harness = false
 
 [[bench]]
 name = "receive_emails"
+required-features = ["internals"]
 harness = false
 
 [[bench]]
@@ -167,34 +169,27 @@ harness = false
 anyhow = "1"
 async-channel = "2.3.1"
 base64 = "0.22"
-chrono = { version = "0.4.38", default-features = false }
+chrono = { version = "0.4.39", default-features = false }
 deltachat-contact-tools = { path = "deltachat-contact-tools" }
 deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
 deltachat = { path = ".", default-features = false }
-futures = "0.3.30"
+futures = "0.3.31"
-futures-lite = "2.3.0"
+futures-lite = "2.5.0"
 libc = "0.2"
 log = "0.4"
 nu-ansi-term = "0.46"
 num-traits = "0.2"
-once_cell = "1.18.0"
+once_cell = "1.20.2"
 rand = "0.8"
 regex = "1.10"
 rusqlite = "0.32"
 sanitize-filename = "0.5"
 serde = "1.0"
 serde_json = "1"
-tempfile = "3.13.0"
+tempfile = "3.14.0"
 thiserror = "1"
-
+tokio = "1"
-# 1.38 is the latest version before `mio` dependency update
+tokio-util = "0.7.13"
-# that broke compilation with Android NDK r23c and r24.
-# Version 1.39.0 cannot be compiled using these NDKs,
-# see issue <https://github.com/tokio-rs/tokio/issues/6748>
-# for details.
-tokio = "~1.38.1"
-
-tokio-util = "0.7.11"
 tracing-subscriber = "0.3"
 yerpc = "0.6.2"
 

README.md

@@ -161,7 +161,6 @@ $ cargo bolero test fuzz_format_flowed --release=false -e afl -s NONE
 ## Features
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
-- `nightly`: Enable nightly only performance and security related features.
 
 ## Update Provider Data
 
@@ -178,8 +177,8 @@ Language bindings are available for:
 
 - **C** \[[📂 source](./deltachat-ffi) | [📚 docs](https://c.delta.chat)\]
 - **Node.js** 
-  - over cffi: \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat)\]
+  - over JSON-RPC: \[[📂 source](./deltachat-rpc-client) | [📦 npm](https://www.npmjs.com/package/@deltachat/jsonrpc-client) | [📚 docs](https://js.jsonrpc.delta.chat/)\]
-  - over jsonrpc built with napi.rs (experimental): \[[📂 source](https://github.com/deltachat/napi-jsonrpc) | [📦 npm](https://www.npmjs.com/package/@deltachat/napi-jsonrpc)\]
+  - over CFFI[^1]: \[[📂 source](./node) | [📦 npm](https://www.npmjs.com/package/deltachat-node) | [📚 docs](https://js.delta.chat/)\]
 - **Python** \[[📂 source](./python) | [📦 pypi](https://pypi.org/project/deltachat) | [📚 docs](https://py.delta.chat)\]
 - **Go**
   - over jsonrpc: \[[📂 source](https://github.com/deltachat/deltachat-rpc-client-go/)\]

RELEASE.md

@@ -14,8 +14,8 @@ For example, to release version 1.116.0 of the core, do the following steps.
 5. Commit the changes as `chore(release): prepare for 1.116.0`.
    Optionally, use a separate branch like `prep-1.116.0` for this commit and open a PR for review.
 
-6. Tag the release: `git tag -a v1.116.0`.
+6. Tag the release: `git tag --annotate v1.116.0`.
 
 7. Push the release tag: `git push origin v1.116.0`.
 
-8. Create a GitHub release: `gh release create v1.116.0 -n ''`.
+8. Create a GitHub release: `gh release create v1.116.0 --notes ''`.

STYLE.md

@@ -0,0 +1,98 @@
+# Coding conventions
+
+We format the code using `rustfmt`. Run `cargo fmt` prior to committing the code.
+Run `scripts/clippy.sh` to check the code for common mistakes with [Clippy].
+
+[Clippy]: https://doc.rust-lang.org/clippy/
+
+## SQL
+
+Multi-line SQL statements should be formatted using string literals,
+for example
+```
+    sql.execute(
+        "CREATE TABLE messages (
+id INTEGER PRIMARY KEY AUTOINCREMENT,
+text TEXT DEFAULT '' NOT NULL -- message text
+) STRICT",
+    )
+    .await?;
+```
+
+Do not use macros like [`concat!`](https://doc.rust-lang.org/std/macro.concat.html)
+or [`indoc!](https://docs.rs/indoc).
+Do not escape newlines like this:
+```
+    sql.execute(
+        "CREATE TABLE messages ( \
+id INTEGER PRIMARY KEY AUTOINCREMENT, \
+text TEXT DEFAULT '' NOT NULL \
+) STRICT",
+    )
+    .await?;
+```
+Escaping newlines
+is prone to errors like this if space before backslash is missing:
+```
+"SELECT foo\
+ FROM bar"
+```
+Literal above results in `SELECT fooFROM bar` string.
+This style also does not allow using `--` comments.
+
+---
+
+Declare new SQL tables with [`STRICT`](https://sqlite.org/stricttables.html) keyword
+to make SQLite check column types.
+
+Declare primary keys with [`AUTOINCREMENT`](https://www.sqlite.org/autoinc.html) keyword.
+This avoids reuse of the row IDs and can avoid dangerous bugs
+like forwarding wrong message because the message was deleted
+and another message took its row ID.
+
+Declare all new columns as `NOT NULL`
+and set the `DEFAULT` value if it is optional so the column can be skipped in `INSERT` statements.
+Dealing with `NULL` values both in SQL and in Rust is tricky and we try to avoid it.
+If column is already declared without `NOT NULL`, use `IFNULL` function to provide default value when selecting it.
+Use `HAVING COUNT(*) > 0` clause
+to [prevent aggregate functions such as `MIN` and `MAX` from returning `NULL`](https://stackoverflow.com/questions/66527856/aggregate-functions-max-etc-return-null-instead-of-no-rows).
+
+Don't delete unused columns too early, but maybe after several months/releases, unused columns are
+still used by older versions, so deleting them breaks downgrading the core or importing a backup in
+an older version. Also don't change the column type, consider adding a new column with another name
+instead. Finally, never change column semantics, this is especially dangerous because the `STRICT`
+keyword doesn't help here.
+
+## Errors
+
+Delta Chat core mostly uses [`anyhow`](https://docs.rs/anyhow/) errors.
+When using [`Context`](https://docs.rs/anyhow/latest/anyhow/trait.Context.html),
+capitalize it but do not add a full stop as the contexts will be separated by `:`.
+For example:
+```
+.with_context(|| format!("Unable to trash message {msg_id}"))
+```
+
+All errors should be handled in one of these ways:
+- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+- With `.log_err().ok()`.
+- Bubbled up with `?`.
+
+`backtrace` feature is enabled for `anyhow` crate
+and `debug = 1` option is set in the test profile.
+This allows to run `RUST_BACKTRACE=1 cargo test`
+and get a backtrace with line numbers in resultified tests
+which return `anyhow::Result`.
+
+## Logging
+
+For logging, use `info!`, `warn!` and `error!` macros.
+Log messages should be capitalized and have a full stop in the end. For example:
+```
+info!(context, "Ignoring addition of {added_addr:?} to {chat_id}.");
+```
+
+Format anyhow errors with `{:#}` to print all the contexts like this:
+```
+error!(context, "Failed to set selfavatar timestamp: {err:#}.");
+```

assets/qr_overlay_delta.svg-part

@@ -0,0 +1,12 @@
+<path
+   style="fill:#ffffff;fill-opacity:1;stroke:none"
+   d="m 24.015419,1.2870249 c -12.549421,0 -22.7283936,10.1789711 -22.7283936,22.7283931 0,12.549422 10.1789726,22.728395 22.7283936,22.728395 14.337742,-0.342877 9.614352,-4.702705 23.697556,0.969161 -7.545453,-13.001555 -1.082973,-13.32964 -0.969161,-23.697556 0,-12.549422 -10.178973,-22.7283931 -22.728395,-22.7283931 z" />
+<path
+   style="fill:#000000;fill-opacity:1;stroke:none"
+   d="M 23.982249,5.3106163 C 13.645822,5.4364005 5.2618355,13.92999 5.2618355,24.275753 c 0,10.345764 8.3839865,18.635301 18.7204135,18.509516 9.827724,-0.03951 7.516769,-5.489695 18.380082,-0.443187 -5.950849,-9.296115 0.201753,-10.533667 0.340336,-18.521947 0,-10.345766 -8.383989,-18.6353031 -18.720418,-18.5095187 z" />
+<g
+    style="fill:#ffffff"
+    transform="scale(1.1342891,0.88160947)">
+  <path
+     d="m 21.360141,23.513382 q -1.218487,-1.364705 -3.387392,-3.265543 -2.388233,-2.095797 -3.216804,-3.289913 -0.828571,-1.218486 -0.828571,-2.6563 0,-2.144536 1.998318,-3.363022 1.998317,-1.2428565 5.215121,-1.2428565 3.216804,0 5.605037,1.0966375 2.412603,1.096638 2.412603,3.021846 0,0.92605 -0.584873,1.535293 -0.584874,0.609243 -1.364705,0.609243 -1.121008,0 -2.631931,-1.681511 -1.535292,-1.705881 -2.60756,-2.388233 -1.047898,-0.706722 -2.461343,-0.706722 -1.803359,0 -2.973106,0.804201 -1.145377,0.804201 -1.145377,2.047057 0,1.169747 0.950419,2.193275 0.950419,1.023529 4.898315,3.728568 4.215963,2.899998 5.946213,4.532769 1.75462,1.632772 2.851258,3.972265 1.096638,2.339494 1.096638,4.947055 0,4.581508 -3.241174,8.090749 -3.216804,3.484871 -7.530245,3.484871 -3.923526,0 -6.628566,-2.802519 -2.705039,-2.802518 -2.705039,-7.481506 0,-4.508399 2.973106,-7.530245 2.997477,-3.021846 7.359658,-3.655459 z m 1.072268,1.121008 q -6.994112,1.145377 -6.994112,9.601672 0,4.36218 1.730251,6.774783 1.75462,2.412603 4.069744,2.412603 2.412603,0 3.972265,-2.315124 1.559663,-2.339493 1.559663,-6.311759 0,-5.751255 -4.337811,-10.162175 z" />
+</g>

benches/receive_emails.rs

@@ -12,18 +12,18 @@ use deltachat::{
 };
 use tempfile::tempdir;
 
-async fn recv_all_emails(context: Context) -> Context {
+async fn recv_all_emails(context: Context, iteration: u32) -> Context {
     for i in 0..100 {
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Mr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Mr.{iteration}.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com
 From: sender@testrun.org
 Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
-In-Reply-To: Mr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Mr.{iteration}.{i_dec}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
@@ -41,11 +41,11 @@ Hello {i}",
 
 /// Receive 100 emails that remove charlie@example.com and add
 /// him back
-async fn recv_groupmembership_emails(context: Context) -> Context {
+async fn recv_groupmembership_emails(context: Context, iteration: u32) -> Context {
     for i in 0..50 {
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Gr.{iteration}.ADD.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
 From: sender@testrun.org
@@ -53,13 +53,12 @@ Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
 Chat-Group-Member-Added: charlie@example.com
-In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Gr.{iteration}.REMOVE.{i_dec}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 
 Hello {i}",
-            i = i,
             i_dec = i - 1,
         );
         receive_imf(&context, black_box(imf_raw.as_bytes()), false)
@@ -68,7 +67,7 @@ Hello {i}",
 
         let imf_raw = format!(
             "Subject: Benchmark
-Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org
+Message-ID: Gr.{iteration}.REMOVE.{i}@testrun.org
 Date: Sat, 07 Dec 2019 19:00:27 +0000
 To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com
 From: sender@testrun.org
@@ -76,14 +75,12 @@ Chat-Version: 1.0
 Chat-Disposition-Notification-To: sender@testrun.org
 Chat-User-Avatar: 0
 Chat-Group-Member-Removed: charlie@example.com
-In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org
+In-Reply-To: Gr.{iteration}.ADD.{i}@testrun.org
 MIME-Version: 1.0
 
 Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
 
-Hello {i}",
+Hello {i}"
-            i = i,
-            i_dec = i - 1,
         );
         receive_imf(&context, black_box(imf_raw.as_bytes()), false)
             .await
@@ -129,11 +126,13 @@ fn criterion_benchmark(c: &mut Criterion) {
     group.bench_function("Receive 100 simple text msgs", |b| {
         let rt = tokio::runtime::Runtime::new().unwrap();
         let context = rt.block_on(create_context());
+        let mut i = 0;
 
         b.to_async(&rt).iter(|| {
             let ctx = context.clone();
+            i += 1;
             async move {
-                recv_all_emails(black_box(ctx)).await;
+                recv_all_emails(black_box(ctx), i).await;
             }
         });
     });
@@ -142,11 +141,13 @@ fn criterion_benchmark(c: &mut Criterion) {
         |b| {
             let rt = tokio::runtime::Runtime::new().unwrap();
             let context = rt.block_on(create_context());
+            let mut i = 0;
 
             b.to_async(&rt).iter(|| {
                 let ctx = context.clone();
+                i += 1;
                 async move {
-                    recv_groupmembership_emails(black_box(ctx)).await;
+                    recv_groupmembership_emails(black_box(ctx), i).await;
                 }
             });
         },

contrib/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

deltachat-contact-tools/src/lib.rs

@@ -15,7 +15,8 @@
     clippy::explicit_into_iter_loop,
     clippy::cloned_instead_of_copied
 )]
-#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 #![allow(
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.147.1"
+version = "1.152.2"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -418,7 +418,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `e2ee_enabled` = 0=no end-to-end-encryption, 1=prefer end-to-end-encryption (default)
  * - `mdns_enabled` = 0=do not send or request read receipts,
  *                    1=send and request read receipts
- *                    default=send and request read receipts, only send but not reuqest if `bot` is set
+ *                    default=send and request read receipts, only send but not request if `bot` is set
  * - `bcc_self`     = 0=do not send a copy of outgoing messages to self,
  *                    1=send a copy of outgoing messages to self (default).
  *                    Sending messages to self is needed for a proper multi-account setup,
@@ -506,6 +506,11 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
+ * - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
+ *                    This is an experimental option not compatible to other MUAs
+ *                    and older Delta Chat versions.
+ *                    1 = enable.
+ *                    0 = disable (default).
  * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
  *                    seconds. 2 days by default.
  *                    This is not supposed to be changed by UIs and only used for testing.
@@ -530,8 +535,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    These keys go to backups and allow easy per-account settings when using @ref dc_accounts_t,
  *                    however, are not handled by the core otherwise.
  * - `webxdc_realtime_enabled` = Whether the realtime APIs should be enabled.
- *                               0 = WebXDC realtime API is disabled and behaves as noop (default).
+ *                               0 = WebXDC realtime API is disabled and behaves as noop.
- *                               1 = WebXDC realtime API is enabled.
+ *                               1 = WebXDC realtime API is enabled (default).
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -717,12 +722,6 @@ char*           dc_get_connectivity_html     (dc_context_t* context);
 int              dc_get_push_state           (dc_context_t* context);
 
 
-/**
- * Only used by the python tests.
- */
-int             dc_all_work_done             (dc_context_t* context);
-
-
 // connect
 
 /**
@@ -964,54 +963,6 @@ uint32_t        dc_create_chat_by_contact_id (dc_context_t* context, uint32_t co
 uint32_t        dc_get_chat_id_by_contact_id (dc_context_t* context, uint32_t contact_id);
 
 
-/**
- * Prepare a message for sending.
- *
- * Call this function if the file to be sent is still in creation.
- * Once you're done with creating the file, call dc_send_msg() as usual
- * and the message will really be sent.
- *
- * This is useful as the user can already send the next messages while
- * e.g. the recoding of a video is not yet finished. Or the user can even forward
- * the message with the file being still in creation to other groups.
- *
- * Files being sent with the increation-method must be placed in the
- * blob directory, see dc_get_blobdir().
- * If the increation-method is not used - which is probably the normal case -
- * dc_send_msg() copies the file to the blob directory if it is not yet there.
- * To distinguish the two cases, msg->state must be set properly. The easiest
- * way to ensure this is to re-use the same object for both calls.
- *
- * Example:
- * ~~~
- * char* blobdir = dc_get_blobdir(context);
- * char* file_to_send = mprintf("%s/%s", blobdir, "send.mp4")
- *
- * dc_msg_t* msg = dc_msg_new(context, DC_MSG_VIDEO);
- * dc_msg_set_file(msg, file_to_send, NULL);
- * dc_prepare_msg(context, chat_id, msg);
- *
- * // ... create the file ...
- *
- * dc_send_msg(context, chat_id, msg);
- *
- * dc_msg_unref(msg);
- * free(file_to_send);
- * dc_str_unref(file_to_send);
- * ~~~
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param chat_id The chat ID to send the message to.
- * @param msg The message object to send to the chat defined by the chat ID.
- *     On success, msg_id and state of the object are set up,
- *     The function does not take ownership of the object,
- *     so you have to free it using dc_msg_unref() as usual.
- * @return The ID of the message that is being prepared.
- */
-uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
-
-
 /**
  * Send a message defined by a dc_msg_t object to a chat.
  *
@@ -1036,13 +987,11 @@ uint32_t        dc_prepare_msg               (dc_context_t* context, uint32_t ch
  * If that fails, is not possible, or the image is already small enough, the image is sent as original.
  * If you want images to be always sent as the original file, use the #DC_MSG_FILE type.
  *
- * Videos and other file types are currently not recoded by the library,
+ * Videos and other file types are currently not recoded by the library.
- * with dc_prepare_msg(), however, you can do that from the UI.
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to send the message to.
- *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
  *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
@@ -1059,7 +1008,6 @@ uint32_t        dc_send_msg                  (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to send the message to.
- *     If dc_prepare_msg() was called before, this parameter can be 0.
  * @param msg The message object to send to the chat defined by the chat ID.
  *     On success, msg_id of the object is set up,
  *     The function does not take ownership of the object,
@@ -1149,9 +1097,14 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id The ID of the message with the webxdc instance.
- * @param json program-readable data, the actual payload
+ * @param json program-readable data, this is created in JS land as:
- * @param descr The user-visible description of JSON data,
+ *     - `payload`: any JS object or primitive.
- *     in case of a chess game, e.g. the move.
+ *     - `info`: optional informational message. Will be shown in chat and may be added as system notification.
+ *       note that also users that are not notified explicitly get the `info` or `summary` update shown in the chat.
+ *     - `document`: optional document name. shown eg. in title bar.
+ *     - `summary`: optional summary. shown beside app icon.
+ *     - `notify`: optional array of other users `selfAddr` to be notified e.g. by a sound about `info` or `summary`.
+ * @param descr Deprecated, set to NULL
  * @return 1=success, 0=error
  */
 int dc_send_webxdc_status_update (dc_context_t* context, uint32_t msg_id, const char* json, const char* descr);
@@ -2533,9 +2486,11 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   ask the user if they want to use the given service for video chats;
  *   if so, call dc_set_config_from_qr().
  *
- * - DC_QR_SOCKS5_PROXY with dc_lot_t::text1=host, dc_lot_t::text2=port:
+ * - DC_QR_PROXY with dc_lot_t::text1=address:
- *   ask the user if they want to use the given proxy and overwrite the previous one, if any.
+ *   ask the user if they want to use the given proxy.
  *   if so, call dc_set_config_from_qr() and restart I/O.
+ *   On success, dc_get_config(context, "proxy_url")
+ *   will contain the new proxy in normalized form as the first element.
  *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
  *   e-mail address scanned, optionally, a draft message could be set in
@@ -2585,13 +2540,15 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
 
 /**
  * Get QR code text that will offer an Setup-Contact or Verified-Group invitation.
- * The QR code is compatible to the OPENPGP4FPR format
- * so that a basic fingerprint comparison also works e.g. with OpenKeychain.
  *
  * The scanning device will pass the scanned content to dc_check_qr() then;
  * if dc_check_qr() returns DC_QR_ASK_VERIFYCONTACT or DC_QR_ASK_VERIFYGROUP
  * an out-of-band-verification can be joined using dc_join_securejoin()
  *
+ * The returned text will also work as a normal https:-link,
+ * so that the QR code is useful also without Delta Chat being installed
+ * or can be passed to contacts through other channels.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id If set to a group-chat-id,
@@ -3977,7 +3934,7 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  *
  * Outgoing message states:
  * - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
- *   the message enters this state before @ref DC_STATE_OUT_PENDING.
+ *   the message enters this state before @ref DC_STATE_OUT_PENDING. Deprecated.
  * - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
  * - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
@@ -4187,9 +4144,13 @@ char*             dc_msg_get_webxdc_blob      (const dc_msg_t* msg, const char*
  *   defaults to an empty string.
  *   Implementations may offer an menu or a button to open this URL.
  * - internet_access:
- *   true if the Webxdc should get full internet access, including Webrtc.
+ *   true if the Webxdc should get internet access;
- *   currently, this is only true for encrypted Webxdc's in the self chat
+ *   this is the case i.e. for experimental maps integration.
- *   that have requested internet access in the manifest.
+ * - self_addr: address to be used for `window.webxdc.selfAddr` in JS land.
+ * - send_update_interval: Milliseconds to wait before calling `sendUpdate()` again since the last call.
+ *   Should be exposed to `webxdc.sendUpdateInterval` in JS land.
+ * - send_update_max_size: Maximum number of bytes accepted for a serialized update object.
+ *   Should be exposed to `webxdc.sendUpdateMaxSize` in JS land.
  *
  * @memberof dc_msg_t
  * @param msg The webxdc instance.
@@ -4475,6 +4436,7 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
  *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
  *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
+ * - DC_INFO_WEBXDC_INFO_MESSAGE (32) - Info-message created by webxdc app sending `update.info`
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4504,19 +4466,23 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
+
 /**
- * Check if a message is still in creation. A message is in creation between
+ * Get link attached to an webxdc info message.
- * the calls to dc_prepare_msg() and dc_send_msg().
+ * The info message needs to be of type DC_INFO_WEBXDC_INFO_MESSAGE.
  *
- * Typically, this is used for videos that are recoded by the UI before
+ * Typically, this is used to set `document.location.href` in JS land.
- * they can be sent.
+ *
+ * Webxdc apps can define the link by setting `update.href` when sending and update,
+ * see dc_send_webxdc_status_update().
  *
  * @memberof dc_msg_t
- * @param msg The message object.
+ * @param msg The info message object.
- * @return 1=message is still in creation (dc_send_msg() was not called yet),
+ *     Not: the webxdc instance.
- *     0=message no longer in creation.
+ * @return The link to be set to `document.location.href` in JS land.
+ *     Returns NULL if there is no link attached to the info message and on errors.
  */
-int             dc_msg_is_increation          (const dc_msg_t* msg);
+char*           dc_msg_get_webxdc_href        (const dc_msg_t* msg);
 
 
 /**
@@ -4669,7 +4635,7 @@ int dc_msg_has_html (dc_msg_t* msg);
   *                                     If the download fails or succeeds,
   *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
   *
-  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any further download action.
   *                                     It was fully downloaded, but we failed to decrypt it.
   * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
@@ -5427,6 +5393,8 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Message containing a sticker, similar to image.
+ * NB: When sending, the message viewtype may be changed to `Image` by some heuristics like checking
+ * for transparent pixels.
  * If possible, the UI should display the image without borders in a transparent way.
  * A click on a sticker will offer to install the sticker set in some future.
  */
@@ -5531,6 +5499,8 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * Outgoing message being prepared. See dc_msg_get_state() for details.
+ *
+ * @deprecated 2024-12-07
  */
 #define         DC_STATE_OUT_PREPARING       18
 
@@ -5704,8 +5674,14 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_CERTCK_STRICT 1
 
 /**
- * Accept invalid certificates, including self-signed ones
+ * Accept certificates that are expired, self-signed
- * or having incorrect hostname.
+ * or not valid for the server hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID 2
+
+/**
+ * For API compatibility only: Treat this as DC_CERTCK_ACCEPT_INVALID on reading.
+ * Must not be written.
  */
 #define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
 
@@ -5745,6 +5721,23 @@ void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);
  * returns immediately and once the result is ready it can be retrieved via dc_jsonrpc_next_response()
  * the jsonrpc specification defines an invocation id that can then be used to match request and response.
  *
+ * An overview of JSON-RPC calls is available at
+ * <https://js.jsonrpc.delta.chat/classes/RawClient.html>.
+ * Note that the page describes only the rough methods.
+ * Calling convention, casing etc. does vary, this is a known flaw,
+ * and at some point we will get to improve that :)
+ *
+ * Also, note that most calls are more high-level than this CFFI, require more database calls and are slower.
+ * They're more suitable for an environment that is totally async and/or cannot use CFFI, which might not be true for native apps.
+ *
+ * Notable exceptions that exist only as JSON-RPC and probably never get a CFFI counterpart:
+ * - getMessageReactions(), sendReaction()
+ * - getHttpResponse()
+ * - draftSelfReport()
+ * - getAccountFileSize()
+ * - importVcard(), parseVcard(), makeVcard()
+ * - sendWebxdcRealtimeData, sendWebxdcRealtimeAdvertisement(), leaveWebxdcRealtime()
+ *
  * @memberof dc_jsonrpc_instance_t
  * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
  * @param request JSON-RPC request as string
@@ -5765,6 +5758,8 @@ char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);
 /**
  * Make a JSON-RPC call and return a response.
  *
+ * See dc_jsonrpc_request() for an overview of possible calls and for more information.
+ *
  * @memberof dc_jsonrpc_instance_t
  * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
  * @param input JSON-RPC request.
@@ -5860,15 +5855,26 @@ int dc_event_get_data2_int(dc_event_t* event);
 
 /**
  * Get data associated with an event object.
- * The meaning of the data depends on the event ID
+ * The meaning of the data depends on the event ID returned as @ref DC_EVENT constants.
- * returned as @ref DC_EVENT constants by dc_event_get_id().
- * See also dc_event_get_data1_int() and dc_event_get_data2_int().
  *
  * @memberof dc_event_t
  * @param event Event object as returned from dc_get_next_event().
- * @return "data2" as a string or NULL.
+ * @return "data1" string or NULL.
- *     the meaning depends on the event type associated with this event.
+ *     The meaning depends on the event type associated with this event.
- *     Once you're done with the string, you have to unref it using dc_unref_str().
+ *     Must be freed using dc_str_unref().
+ */
+char* dc_event_get_data1_str(dc_event_t* event);
+
+
+/**
+ * Get data associated with an event object.
+ * The meaning of the data depends on the event ID returned as @ref DC_EVENT constants.
+ *
+ * @memberof dc_event_t
+ * @param event Event object as returned from dc_get_next_event().
+ * @return "data2" string or NULL.
+ *     The meaning depends on the event type associated with this event.
+ *     Must be freed using dc_str_unref().
  */
 char* dc_event_get_data2_str(dc_event_t* event);
 
@@ -6066,12 +6072,35 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_INCOMING_REACTION        2002
 
 
+
+/**
+ * A webxdc wants an info message or a changed summary to be notified.
+ *
+ * @param data1 (int) contact_id ID _and_ (char*) href.
+ *      - dc_event_get_data1_int() returns contact_id of the sending contact.
+ *      - dc_event_get_data1_str() returns the href as set to `update.href`.
+ * @param data2 (int) msg_id _and_ (char*) text_to_notify.
+ *      - dc_event_get_data2_int() returns the msg_id,
+ *        referring to the webxdc-info-message, if there is any.
+ *        Sometimes no webxdc-info-message is added to the chat
+ *        and yet a notification is sent; in this case the msg_id
+ *        of the webxdc instance is returned.
+ *      - dc_event_get_data2_str() returns text_to_notify,
+ *        the text that shall be shown in the notification.
+ *        string must be passed to dc_str_unref() afterwards.
+ */
+#define DC_EVENT_INCOMING_WEBXDC_NOTIFY   2003
+
+
 /**
  * There is a fresh message. Typically, the user will show an notification
  * when receiving this message.
  *
  * There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
  *
+ * If the message is a webxdc info message,
+ * dc_msg_get_parent() returns the webxdc instance the notification belongs to.
+ *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
  */
@@ -6355,6 +6384,25 @@ void dc_event_unref(dc_event_t* event);
 
 #define DC_EVENT_CHATLIST_ITEM_CHANGED         2301
 
+/**
+ * Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+ *
+ * This event is only emitted by the account manager.
+ */
+
+#define DC_EVENT_ACCOUNTS_CHANGED              2302
+
+/**
+ * Inform that an account property that might be shown in the account list changed, namely:
+ * - is_configured (see dc_is_configured())
+ * - displayname
+ * - selfavatar
+ * - private_tag
+ * 
+ * This event is emitted from the account whose property changed.
+ */
+
+#define DC_EVENT_ACCOUNTS_ITEM_CHANGED         2303
 
 /**
  * Inform that some events have been skipped due to event channel overflow.
@@ -6802,7 +6850,7 @@ void dc_event_unref(dc_event_t* event);
 
 /// "Failed to send message to %1$s."
 ///
-/// Used in status messages.
+/// Unused. Was used in group chat status messages.
 /// - %1$s will be replaced by the name of the contact the message cannot be sent to
 #define DC_STR_FAILED_SENDING_TO          74
 

deltachat-ffi/src/lib.rs

@@ -413,16 +413,6 @@ pub unsafe extern "C" fn dc_get_push_state(context: *const dc_context_t) -> libc
     block_on(ctx.push_state()) as libc::c_int
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_all_work_done(context: *mut dc_context_t) -> libc::c_int {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_all_work_done()");
-        return 0;
-    }
-    let ctx = &*context;
-    block_on(async move { ctx.all_work_done().await as libc::c_int })
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_oauth2_url(
     context: *mut dc_context_t,
@@ -542,6 +532,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::MsgsChanged { .. } => 2000,
         EventType::ReactionsChanged { .. } => 2001,
         EventType::IncomingReaction { .. } => 2002,
+        EventType::IncomingWebxdcNotify { .. } => 2003,
         EventType::IncomingMsg { .. } => 2005,
         EventType::IncomingMsgBunch { .. } => 2006,
         EventType::MsgsNoticed { .. } => 2008,
@@ -568,6 +559,8 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::AccountsBackgroundFetchDone => 2200,
         EventType::ChatlistChanged => 2300,
         EventType::ChatlistItemChanged { .. } => 2301,
+        EventType::AccountsChanged => 2302,
+        EventType::AccountsItemChanged => 2303,
         EventType::EventChannelOverflow { .. } => 2400,
         #[allow(unreachable_patterns)]
         #[cfg(test)]
@@ -600,9 +593,12 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ErrorSelfNotInGroup(_)
-        | EventType::AccountsBackgroundFetchDone => 0,
+        | EventType::AccountsBackgroundFetchDone
-        EventType::ChatlistChanged => 0,
+        | EventType::ChatlistChanged
-        EventType::IncomingReaction { contact_id, .. } => contact_id.to_u32() as libc::c_int,
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged => 0,
+        EventType::IncomingReaction { contact_id, .. }
+        | EventType::IncomingWebxdcNotify { contact_id, .. } => contact_id.to_u32() as libc::c_int,
         EventType::MsgsChanged { chat_id, .. }
         | EventType::ReactionsChanged { chat_id, .. }
         | EventType::IncomingMsg { chat_id, .. }
@@ -674,6 +670,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::AccountsBackgroundFetchDone
         | EventType::ChatlistChanged
         | EventType::ChatlistItemChanged { .. }
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged
         | EventType::ConfigSynced { .. }
         | EventType::ChatModified(_)
         | EventType::WebxdcRealtimeAdvertisementReceived { .. }
@@ -681,6 +679,7 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
         | EventType::IncomingReaction { msg_id, .. }
+        | EventType::IncomingWebxdcNotify { msg_id, .. }
         | EventType::IncomingMsg { msg_id, .. }
         | EventType::MsgDelivered { msg_id, .. }
         | EventType::MsgFailed { msg_id, .. }
@@ -700,6 +699,27 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_event_get_data1_str(event: *mut dc_event_t) -> *mut libc::c_char {
+    if event.is_null() {
+        eprintln!("ignoring careless call to dc_event_get_data1_str()");
+        return ptr::null_mut();
+    }
+
+    let event = &(*event).typ;
+
+    match event {
+        EventType::IncomingWebxdcNotify { href, .. } => {
+            if let Some(href) = href {
+                href.to_c_string().unwrap_or_default().into_raw()
+            } else {
+                ptr::null_mut()
+            }
+        }
+        _ => ptr::null_mut(),
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut libc::c_char {
     if event.is_null() {
@@ -748,6 +768,8 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::IncomingMsgBunch { .. }
         | EventType::ChatlistItemChanged { .. }
         | EventType::ChatlistChanged
+        | EventType::AccountsChanged
+        | EventType::AccountsItemChanged
         | EventType::WebxdcRealtimeAdvertisementReceived { .. }
         | EventType::EventChannelOverflow { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
@@ -775,6 +797,9 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             .to_c_string()
             .unwrap_or_default()
             .into_raw(),
+        EventType::IncomingWebxdcNotify { text, .. } => {
+            text.to_c_string().unwrap_or_default().into_raw()
+        }
         #[allow(unreachable_patterns)]
         #[cfg(test)]
         _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
@@ -951,27 +976,6 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     })
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_prepare_msg(
-    context: *mut dc_context_t,
-    chat_id: u32,
-    msg: *mut dc_msg_t,
-) -> u32 {
-    if context.is_null() || chat_id == 0 || msg.is_null() {
-        eprintln!("ignoring careless call to dc_prepare_msg()");
-        return 0;
-    }
-    let ctx = &mut *context;
-    let ffi_msg: &mut MessageWrapper = &mut *msg;
-
-    block_on(async move {
-        chat::prepare_msg(ctx, ChatId::new(chat_id), &mut ffi_msg.message)
-            .await
-            .unwrap_or_log_default(ctx, "Failed to prepare message")
-    })
-    .to_u32()
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_send_msg(
     context: *mut dc_context_t,
@@ -1059,7 +1063,7 @@ pub unsafe extern "C" fn dc_send_webxdc_status_update(
     context: *mut dc_context_t,
     msg_id: u32,
     json: *const libc::c_char,
-    descr: *const libc::c_char,
+    _descr: *const libc::c_char,
 ) -> libc::c_int {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_send_webxdc_status_update()");
@@ -1067,11 +1071,7 @@ pub unsafe extern "C" fn dc_send_webxdc_status_update(
     }
     let ctx = &*context;
 
-    block_on(ctx.send_webxdc_status_update(
+    block_on(ctx.send_webxdc_status_update(MsgId::new(msg_id), &to_string_lossy(json)))
-        MsgId::new(msg_id),
-        &to_string_lossy(json),
-        &to_string_lossy(descr),
-    ))
         .context("Failed to send webxdc update")
         .log_err(ctx)
         .is_ok() as libc::c_int
@@ -3682,13 +3682,14 @@ pub unsafe extern "C" fn dc_msg_get_info_type(msg: *mut dc_msg_t) -> libc::c_int
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_msg_is_increation(msg: *mut dc_msg_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_msg_get_webxdc_href(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
-        eprintln!("ignoring careless call to dc_msg_is_increation()");
+        eprintln!("ignoring careless call to dc_msg_get_webxdc_href()");
-        return 0;
+        return "".strdup();
     }
+
     let ffi_msg = &*msg;
-    ffi_msg.message.is_increation().into()
+    ffi_msg.message.get_webxdc_href().strdup()
 }
 
 #[no_mangle]
@@ -4868,7 +4869,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     }
 
     let accounts = &*accounts;
-    block_on(async move { accounts.write().await.maybe_network_lost().await });
+    block_on(async move { accounts.read().await.maybe_network_lost().await });
 }
 
 #[no_mangle]
@@ -4882,12 +4883,12 @@ pub unsafe extern "C" fn dc_accounts_background_fetch(
     }
 
     let accounts = &*accounts;
-    block_on(async move {
+    let background_fetch_future = {
-        let accounts = accounts.read().await;
+        let lock = block_on(accounts.read());
-        accounts
+        lock.background_fetch(Duration::from_secs(timeout_in_seconds))
-            .background_fetch(Duration::from_secs(timeout_in_seconds))
+    };
-            .await;
+    // At this point account manager is not locked anymore.
-    });
+    block_on(background_fetch_future);
     1
 }
 
@@ -4905,7 +4906,7 @@ pub unsafe extern "C" fn dc_accounts_set_push_device_token(
     let token = to_string_lossy(token);
 
     block_on(async move {
-        let mut accounts = accounts.write().await;
+        let accounts = accounts.read().await;
         if let Err(err) = accounts.set_push_device_token(&token).await {
             accounts.emit_event(EventType::Error(format!(
                 "Failed to set notify token: {err:#}."

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.147.1"
+version = "1.152.2"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -25,7 +25,7 @@ async-channel = { workspace = true }
 futures = { workspace = true }
 serde_json = { workspace = true }
 yerpc = { workspace = true, features = ["anyhow_expose", "openrpc"] }
-typescript-type-def = { version = "0.5.12", features = ["json_value"] }
+typescript-type-def = { version = "0.5.13", features = ["json_value"] }
 tokio = { workspace = true }
 sanitize-filename = { workspace = true }
 walkdir = "2.5.0"
@@ -33,7 +33,7 @@ base64 = { workspace = true }
 
 # optional dependencies
 axum = { version = "0.7", optional = true, features = ["ws"] }
-env_logger = { version = "0.11.5", optional = true }
+env_logger = { version = "0.11.6", optional = true }
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["full", "rt-multi-thread"] }

deltachat-jsonrpc/src/api.rs

@@ -254,11 +254,12 @@ impl CommandApi {
     /// Process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
     async fn accounts_background_fetch(&self, timeout_in_seconds: f64) -> Result<()> {
-        self.accounts
+        let future = {
-            .write()
+            let lock = self.accounts.read().await;
-            .await
+            lock.background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
-            .background_fetch(std::time::Duration::from_secs_f64(timeout_in_seconds))
+        };
-            .await;
+        // At this point account manager is not locked anymore.
+        future.await;
         Ok(())
     }
 
@@ -1134,9 +1135,11 @@ impl CommandApi {
     async fn get_message(&self, account_id: u32, msg_id: u32) -> Result<MessageObject> {
         let ctx = self.get_context(account_id).await?;
         let msg_id = MsgId::new(msg_id);
-        MessageObject::from_msg_id(&ctx, msg_id)
+        let message_object = MessageObject::from_msg_id(&ctx, msg_id)
             .await
-            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))
+            .with_context(|| format!("Failed to load message {msg_id} for account {account_id}"))?
+            .with_context(|| format!("Message {msg_id} does not exist for account {account_id}"))?;
+        Ok(message_object)
     }
 
     async fn get_message_html(&self, account_id: u32, message_id: u32) -> Result<Option<String>> {
@@ -1160,7 +1163,10 @@ impl CommandApi {
             messages.insert(
                 message_id,
                 match message_result {
-                    Ok(message) => MessageLoadResult::Message(message),
+                    Ok(Some(message)) => MessageLoadResult::Message(message),
+                    Ok(None) => MessageLoadResult::LoadingError {
+                        error: "Message does not exist".to_string(),
+                    },
                     Err(error) => MessageLoadResult::LoadingError {
                         error: format!("{error:#}"),
                     },
@@ -1418,6 +1424,15 @@ impl CommandApi {
         Ok(())
     }
 
+    /// Resets contact encryption.
+    async fn reset_contact_encryption(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_id = ContactId::new(contact_id);
+
+        contact_id.reset_encryption(&ctx).await?;
+        Ok(())
+    }
+
     async fn change_contact_name(
         &self,
         account_id: u32,
@@ -1752,10 +1767,10 @@ impl CommandApi {
         account_id: u32,
         instance_msg_id: u32,
         update_str: String,
-        description: String,
+        _descr: Option<String>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        ctx.send_webxdc_status_update(MsgId::new(instance_msg_id), &update_str, &description)
+        ctx.send_webxdc_status_update(MsgId::new(instance_msg_id), &update_str)
             .await
     }
 
@@ -1814,6 +1829,18 @@ impl CommandApi {
         WebxdcMessageInfo::get_for_message(&ctx, MsgId::new(instance_msg_id)).await
     }
 
+    /// Get href from a WebxdcInfoMessage which might include a hash holding
+    /// information about a specific position or state in a webxdc app (optional)
+    async fn get_webxdc_href(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+    ) -> Result<Option<String>> {
+        let ctx = self.get_context(account_id).await?;
+        let message = Message::load_from_db(&ctx, MsgId::new(instance_msg_id)).await?;
+        Ok(message.get_webxdc_href())
+    }
+
     /// Get blob encoded as base64 from a webxdc message
     ///
     /// path is the path of the file within webxdc archive
@@ -1989,9 +2016,7 @@ impl CommandApi {
     async fn get_draft(&self, account_id: u32, chat_id: u32) -> Result<Option<MessageObject>> {
         let ctx = self.get_context(account_id).await?;
         if let Some(draft) = ChatId::new(chat_id).get_draft(&ctx).await? {
-            Ok(Some(
+            Ok(MessageObject::from_msg_id(&ctx, draft.get_id()).await?)
-                MessageObject::from_msg_id(&ctx, draft.get_id()).await?,
-            ))
         } else {
             Ok(None)
         }
@@ -2117,8 +2142,7 @@ impl CommandApi {
     ) -> Result<u32> {
         let ctx = self.get_context(account_id).await?;
 
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text(text);
-        msg.set_text(text);
 
         let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
         Ok(message_id.to_u32())
@@ -2161,7 +2185,9 @@ impl CommandApi {
                 .await?;
         }
         let msg_id = chat::send_msg(&ctx, ChatId::new(chat_id), &mut message).await?;
-        let message = MessageObject::from_msg_id(&ctx, msg_id).await?;
+        let message = MessageObject::from_msg_id(&ctx, msg_id)
+            .await?
+            .context("Just sent message does not exist")?;
         Ok((msg_id.to_u32(), message))
     }
 

deltachat-jsonrpc/src/api/types/account.rs

@@ -17,6 +17,9 @@ pub enum Account {
         // size: u32,
         profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
         color: String,
+        /// Optional tag as "Work", "Family".
+        /// Meant to help profile owner to differ between profiles with similar names.
+        private_tag: Option<String>,
     },
     #[serde(rename_all = "camelCase")]
     Unconfigured { id: u32 },
@@ -31,12 +34,14 @@ impl Account {
             let color = color_int_to_hex_string(
                 Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
             );
+            let private_tag = ctx.get_config(Config::PrivateTag).await?;
             Ok(Account::Configured {
                 id,
                 display_name,
                 addr,
                 profile_image,
                 color,
+                private_tag,
             })
         } else {
             Ok(Account::Unconfigured { id })

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -88,11 +88,17 @@ pub(crate) async fn get_chat_list_item_by_id(
 
     let (last_updated, message_type) = match last_msgid {
         Some(id) => {
-            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
+            if let Some(last_message) =
+                deltachat::message::Message::load_from_db_optional(ctx, id).await?
+            {
                 (
                     Some(last_message.get_timestamp() * 1000),
                     Some(last_message.get_viewtype().into()),
                 )
+            } else {
+                // Message may be deleted by the time we try to load it.
+                (None, None)
+            }
         }
         None => (None, None),
     };

deltachat-jsonrpc/src/api/types/events.rs

@@ -69,7 +69,7 @@ pub enum EventType {
     /// or for functions that are expected to fail (eg. autocryptContinueKeyTransfer())
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
-    /// in a messasge box then.
+    /// in a message box then.
     Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
@@ -106,6 +106,16 @@ pub enum EventType {
         reaction: String,
     },
 
+    /// Incoming webxdc info or summary update, should be notified.
+    #[serde(rename_all = "camelCase")]
+    IncomingWebxdcNotify {
+        chat_id: u32,
+        contact_id: u32,
+        msg_id: u32,
+        text: String,
+        href: Option<String>,
+    },
+
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -277,6 +287,20 @@ pub enum EventType {
     #[serde(rename_all = "camelCase")]
     ChatlistItemChanged { chat_id: Option<u32> },
 
+    /// Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+    ///
+    /// This event is only emitted by the account manager
+    AccountsChanged,
+
+    /// Inform that an account property that might be shown in the account list changed, namely:
+    /// - is_configured (see is_configured())
+    /// - displayname
+    /// - selfavatar
+    /// - private_tag
+    ///
+    /// This event is emitted from the account whose property changed.
+    AccountsItemChanged,
+
     /// Inform than some events have been skipped due to event channel overflow.
     EventChannelOverflow { n: u64 },
 }
@@ -319,6 +343,19 @@ impl From<CoreEventType> for EventType {
                 msg_id: msg_id.to_u32(),
                 reaction: reaction.as_str().to_string(),
             },
+            CoreEventType::IncomingWebxdcNotify {
+                chat_id,
+                contact_id,
+                msg_id,
+                text,
+                href,
+            } => IncomingWebxdcNotify {
+                chat_id: chat_id.to_u32(),
+                contact_id: contact_id.to_u32(),
+                msg_id: msg_id.to_u32(),
+                text,
+                href,
+            },
             CoreEventType::IncomingMsg { chat_id, msg_id } => IncomingMsg {
                 chat_id: chat_id.to_u32(),
                 msg_id: msg_id.to_u32(),
@@ -409,6 +446,11 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ChatlistChanged => ChatlistChanged,
             CoreEventType::EventChannelOverflow { n } => EventChannelOverflow { n },
+            CoreEventType::AccountsChanged => AccountsChanged,
+            CoreEventType::AccountsItemChanged => AccountsItemChanged,
+            #[allow(unreachable_patterns)]
+            #[cfg(test)]
+            _ => unreachable!("This is just to silence a rust_analyzer false-positive"),
         }
     }
 }

deltachat-jsonrpc/src/api/types/message.rs

@@ -85,6 +85,8 @@ pub struct MessageObject {
 
     webxdc_info: Option<WebxdcMessageInfo>,
 
+    webxdc_href: Option<String>,
+
     download_state: DownloadState,
 
     reactions: Option<JSONRPCReactions>,
@@ -112,8 +114,10 @@ enum MessageQuote {
 }
 
 impl MessageObject {
-    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
+    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Option<Self>> {
-        let message = Message::load_from_db(context, msg_id).await?;
+        let Some(message) = Message::load_from_db_optional(context, msg_id).await? else {
+            return Ok(None);
+        };
 
         let sender_contact = Contact::get_by_id(context, message.get_from_id())
             .await
@@ -183,7 +187,7 @@ impl MessageObject {
             .map(Into::into)
             .collect();
 
-        Ok(MessageObject {
+        let message_object = MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
             from_id: message.get_from_id().to_u32(),
@@ -239,12 +243,17 @@ impl MessageObject {
             file_name: message.get_filename(),
             webxdc_info,
 
+            // On a WebxdcInfoMessage this might include a hash holding
+            // information about a specific position or state in a webxdc app
+            webxdc_href: message.get_webxdc_href(),
+
             download_state,
 
             reactions,
 
             vcard_contact: vcard_contacts.first().cloned(),
-        })
+        };
+        Ok(Some(message_object))
     }
 }
 
@@ -264,6 +273,9 @@ pub enum MessageViewtype {
     Gif,
 
     /// Message containing a sticker, similar to image.
+    /// NB: When sending, the message viewtype may be changed to `Image` by some heuristics like
+    /// checking for transparent pixels. Use `Message::force_sticker()` to disable them.
+    ///
     /// If possible, the ui should display the image without borders in a transparent way.
     /// A click on a sticker will offer to install the sticker set in some future.
     Sticker,
@@ -490,6 +502,7 @@ pub struct MessageSearchResult {
     author_name: String,
     author_color: String,
     author_id: u32,
+    chat_id: u32,
     chat_profile_image: Option<String>,
     chat_color: String,
     chat_name: String,
@@ -529,6 +542,7 @@ impl MessageSearchResult {
             author_name,
             author_color: color_int_to_hex_string(sender.get_color()),
             author_id: sender.id.to_u32(),
+            chat_id: chat.id.to_u32(),
             chat_name: chat.get_name().to_owned(),
             chat_color,
             chat_type: chat.get_type().to_u32().context("unknown chat type id")?,

deltachat-jsonrpc/src/api/types/qr.rs

@@ -6,87 +6,161 @@ use typescript_type_def::TypeDef;
 #[serde(rename = "Qr", rename_all = "camelCase")]
 #[serde(tag = "kind")]
 pub enum QrObject {
+    /// Ask the user whether to verify the contact.
+    ///
+    /// If the user agrees, pass this QR code to [`crate::securejoin::join_securejoin`].
     AskVerifyContact {
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user whether to join the group.
     AskVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// ID of the contact.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Contact fingerprint is verified.
+    ///
+    /// Ask the user if they want to start chatting.
     FprOk {
+        /// Contact ID.
         contact_id: u32,
     },
+    /// Scanned fingerprint does not match the last seen fingerprint.
     FprMismatch {
+        /// Contact ID.
         contact_id: Option<u32>,
     },
+    /// The scanned QR code contains a fingerprint but no e-mail address.
     FprWithoutAddr {
+        /// Key fingerprint.
         fingerprint: String,
     },
+    /// Ask the user if they want to create an account on the given domain.
     Account {
+        /// Server domain name.
         domain: String,
     },
+    /// Provides a backup that can be retrieved using iroh-net based backup transfer protocol.
     Backup2 {
+        /// Authentication token.
         auth_token: String,
-
+        /// Iroh node address.
         node_addr: String,
     },
+    /// Ask the user if they want to use the given service for video chats.
     WebrtcInstance {
         domain: String,
         instance_pattern: String,
     },
+    /// Ask the user if they want to use the given proxy.
+    ///
+    /// Note that HTTP(S) URLs without a path
+    /// and query parameters are treated as HTTP(S) proxy URL.
+    /// UI may want to still offer to open the URL
+    /// in the browser if QR code contents
+    /// starts with `http://` or `https://`
+    /// and the QR code was not scanned from
+    /// the proxy configuration screen.
     Proxy {
+        /// Proxy URL.
+        ///
+        /// This is the URL that is going to be added.
         url: String,
+        /// Host extracted from the URL to display in the UI.
         host: String,
+        /// Port extracted from the URL to display in the UI.
         port: u16,
     },
+    /// Contact address is scanned.
+    ///
+    /// Optionally, a draft message could be provided.
+    /// Ask the user if they want to start chatting.
     Addr {
+        /// Contact ID.
         contact_id: u32,
+        /// Draft message.
         draft: Option<String>,
     },
-    Url {
+    /// URL scanned.
-        url: String,
+    ///
-    },
+    /// Ask the user if they want to open a browser or copy the URL to clipboard.
-    Text {
+    Url { url: String },
-        text: String,
+    /// Text scanned.
-    },
+    ///
+    /// Ask the user if they want to copy the text to clipboard.
+    Text { text: String },
+    /// Ask the user if they want to withdraw their own QR code.
     WithdrawVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to withdraw their own group invite QR code.
     WithdrawVerifyGroup {
+        /// Group name.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own QR code.
     ReviveVerifyContact {
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
+    /// Ask the user if they want to revive their own group invite QR code.
     ReviveVerifyGroup {
+        /// Contact ID.
         grpname: String,
+        /// Group ID.
         grpid: String,
+        /// Contact ID.
         contact_id: u32,
+        /// Fingerprint of the contact key as scanned from the QR code.
         fingerprint: String,
+        /// Invite number.
         invitenumber: String,
+        /// Authentication code.
         authcode: String,
     },
-    Login {
+    /// `dclogin:` scheme parameters.
-        address: String,
+    ///
-    },
+    /// Ask the user if they want to login with the email address.
+    Login { address: String },
 }
 
 impl From<Qr> for QrObject {
@@ -141,7 +215,6 @@ impl From<Qr> for QrObject {
                 auth_token,
             } => QrObject::Backup2 {
                 node_addr: serde_json::to_string(node_addr).unwrap_or_default(),
-
                 auth_token,
             },
             Qr::WebrtcInstance {

deltachat-jsonrpc/src/api/types/webxdc.rs

@@ -35,6 +35,14 @@ pub struct WebxdcMessageInfo {
     source_code_url: Option<String>,
     /// True if full internet access should be granted to the app.
     internet_access: bool,
+    /// Address to be used for `window.webxdc.selfAddr` in JS land.
+    self_addr: String,
+    /// Milliseconds to wait before calling `sendUpdate()` again since the last call.
+    /// Should be exposed to `window.sendUpdateInterval` in JS land.
+    send_update_interval: usize,
+    /// Maximum number of bytes accepted for a serialized update object.
+    /// Should be exposed to `window.sendUpdateMaxSize` in JS land.
+    send_update_max_size: usize,
 }
 
 impl WebxdcMessageInfo {
@@ -49,7 +57,11 @@ impl WebxdcMessageInfo {
             document,
             summary,
             source_code_url,
+            request_integration: _,
             internet_access,
+            self_addr,
+            send_update_interval,
+            send_update_max_size,
         } = message.get_webxdc_info(context).await?;
 
         Ok(Self {
@@ -59,6 +71,9 @@ impl WebxdcMessageInfo {
             summary: maybe_empty_string_to_option(summary),
             source_code_url: maybe_empty_string_to_option(source_code_url),
             internet_access,
+            self_addr,
+            send_update_interval,
+            send_update_max_size,
         })
     }
 }

deltachat-jsonrpc/src/lib.rs

@@ -1,4 +1,6 @@
 #![recursion_limit = "256"]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 pub mod api;
 pub use yerpc;
 

deltachat-jsonrpc/typescript/package.json

@@ -58,5 +58,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.147.1"
+  "version": "1.152.2"
 }

deltachat-ratelimit/src/lib.rs

@@ -90,6 +90,11 @@ impl Ratelimit {
     pub fn until_can_send(&self) -> Duration {
         self.until_can_send_at(SystemTime::now())
     }
+
+    /// Returns minimum possible update interval in milliseconds.
+    pub fn update_interval(&self) -> usize {
+        (self.window.as_millis() as f64 / self.quota) as usize
+    }
 }
 
 #[cfg(test)]
@@ -102,6 +107,7 @@ mod tests {
 
         let mut ratelimit = Ratelimit::new_at(Duration::new(60, 0), 3.0, now);
         assert!(ratelimit.can_send_at(now));
+        assert_eq!(ratelimit.update_interval(), 20_000);
 
         // Send burst of 3 messages.
         ratelimit.send_at(now);

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.147.1"
+version = "1.152.2"
 license = "MPL-2.0"
 edition = "2021"
 repository = "https://github.com/deltachat/deltachat-core-rust"

deltachat-repl/src/cmdline.rs

@@ -969,9 +969,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 "Arguments <msg-id> <json status update> expected"
             );
             let msg_id = MsgId::new(arg1.parse()?);
-            context
+            context.send_webxdc_status_update(msg_id, arg2).await?;
-                .send_webxdc_status_update(msg_id, arg2, "this is a webxdc status update")
-                .await?;
         }
         "videochat" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
@@ -1004,8 +1002,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             if !arg1.is_empty() {
-                let mut draft = Message::new(Viewtype::Text);
+                let mut draft = Message::new_text(arg1.to_string());
-                draft.set_text(arg1.to_string());
                 sel_chat
                     .as_ref()
                     .unwrap()
@@ -1028,8 +1025,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty(),
                 "Please specify text to add as device message."
             );
-            let mut msg = Message::new(Viewtype::Text);
+            let mut msg = Message::new_text(arg1.to_string());
-            msg.set_text(arg1.to_string());
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "listmedia" => {

deltachat-rpc-client/README.md

@@ -25,7 +25,8 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
+2. Install tox `pip install -U tox`
+3. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 

deltachat-rpc-client/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat-rpc-client"
-version = "1.147.1"
+version = "1.152.2"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -24,9 +24,6 @@ classifiers = [
     "Topic :: Communications :: Email"
 ]
 readme = "README.md"
-dependencies = [
-    "imap-tools",
-]
 
 [tool.setuptools.package-data]
 deltachat_rpc_client = [

deltachat-rpc-client/src/deltachat_rpc_client/const.py

@@ -41,6 +41,7 @@ class EventType(str, Enum):
     REACTIONS_CHANGED = "ReactionsChanged"
     INCOMING_MSG = "IncomingMsg"
     INCOMING_MSG_BUNCH = "IncomingMsgBunch"
+    INCOMING_REACTION = "IncomingReaction"
     MSGS_NOTICED = "MsgsNoticed"
     MSG_DELIVERED = "MsgDelivered"
     MSG_FAILED = "MsgFailed"
@@ -61,8 +62,11 @@ class EventType(str, Enum):
     WEBXDC_INSTANCE_DELETED = "WebxdcInstanceDeleted"
     CHATLIST_CHANGED = "ChatlistChanged"
     CHATLIST_ITEM_CHANGED = "ChatlistItemChanged"
+    ACCOUNTS_CHANGED = "AccountsChanged"
+    ACCOUNTS_ITEM_CHANGED = "AccountsItemChanged"
     CONFIG_SYNCED = "ConfigSynced"
     WEBXDC_REALTIME_DATA = "WebxdcRealtimeData"
+    WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED = "WebxdcRealtimeAdvertisementReceived"
 
 
 class ChatId(IntEnum):

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -36,6 +36,10 @@ class Contact:
         """Delete contact."""
         self._rpc.delete_contact(self.account.id, self.id)
 
+    def reset_encryption(self) -> None:
+        """Reset contact encryption."""
+        self._rpc.reset_contact_encryption(self.account.id, self.id)
+
     def set_name(self, name: str) -> None:
         """Change the name of this contact."""
         self._rpc.change_contact_name(self.account.id, self.id, name)

deltachat-rpc-client/tests/conftest.py

@@ -1,7 +1,3 @@
-"""
-Internal Python-level IMAP handling used by the tests.
-"""
-
 from __future__ import annotations
 
 import imaplib
@@ -11,17 +7,11 @@ import ssl
 from contextlib import contextmanager
 from typing import TYPE_CHECKING
 
-from imap_tools import (
+import pytest
-    AND,
+from imap_tools import AND, Header, MailBox, MailMessage, MailMessageFlags, errors
-    Header,
-    MailBox,
-    MailMessage,
-    MailMessageFlags,
-    errors,
-)
 
 if TYPE_CHECKING:
-    from . import Account
+    from deltachat_rpc_client import Account
 
 FLAGS = b"FLAGS"
 FETCH = b"FETCH"
@@ -29,6 +19,8 @@ ALL = "1:*"
 
 
 class DirectImap:
+    """Internal Python-level IMAP handling."""
+
     def __init__(self, account: Account) -> None:
         self.account = account
         self.logid = account.get_config("displayname") or id(account)
@@ -212,3 +204,8 @@ class IdleManager:
     def done(self):
         """send idle-done to server if we are currently in idle mode."""
         return self.direct_imap.conn.idle.stop()
+
+
+@pytest.fixture
+def direct_imap():
+    return DirectImap

deltachat-rpc-client/tests/test_account_events.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from deltachat_rpc_client import EventType
+
+if TYPE_CHECKING:
+    from deltachat_rpc_client.pytestplugin import ACFactory
+
+
+def test_event_on_configuration(acfactory: ACFactory) -> None:
+    """
+    Test if ACCOUNTS_ITEM_CHANGED event is emitted on configure
+    """
+
+    account = acfactory.new_preconfigured_account()
+    account.clear_all_events()
+    assert not account.is_configured()
+    future = account.configure.future()
+    while True:
+        event = account.wait_for_event()
+        if event.kind == EventType.ACCOUNTS_ITEM_CHANGED:
+            break
+    assert account.is_configured()
+
+    future()
+
+
+# other tests are written in rust: src/tests/account_events.rs

deltachat-rpc-client/tests/test_iroh_webxdc.py

@@ -7,7 +7,8 @@ If you want to debug iroh at rust-trace/log level set
     RUST_LOG=iroh_net=trace,iroh_gossip=trace
 """
 
-import sys
+import logging
+import os
 import threading
 import time
 
@@ -24,9 +25,7 @@ def path_to_webxdc(request):
 
 
 def log(msg):
-    print()
+    logging.info(msg)
-    print("*" * 80 + "\n" + msg + "\n", file=sys.stderr)
-    print()
 
 
 def setup_realtime_webxdc(ac1, ac2, path_to_webxdc):
@@ -107,13 +106,15 @@ def test_realtime_sequentially(acfactory, path_to_webxdc):
     assert snapshot.text == "ping2"
 
     log("sending realtime data ac1 -> ac2")
-    ac1_webxdc_msg.send_webxdc_realtime_data(b"foo")
+    # Test that 128 KB of data can be sent in a single message.
+    data = os.urandom(128000)
+    ac1_webxdc_msg.send_webxdc_realtime_data(data)
 
     log("ac2: waiting for realtime data")
     while 1:
         event = ac2.wait_for_event()
         if event.kind == EventType.WEBXDC_REALTIME_DATA:
-            assert event.data == list(b"foo")
+            assert event.data == list(data)
             break
 
 
@@ -208,3 +209,28 @@ def test_no_reordering(acfactory, path_to_webxdc):
                 if event.data[0] == i:
                     break
                 pytest.fail("Reordering detected")
+
+
+def test_advertisement_after_chatting(acfactory, path_to_webxdc):
+    """Test that realtime advertisement is assigned to the correct message after chatting."""
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("webxdc_realtime_enabled", "1")
+    ac2.set_config("webxdc_realtime_enabled", "1")
+
+    ac1_ac2_chat = ac1.create_chat(ac2)
+    ac1_webxdc_msg = ac1_ac2_chat.send_message(text="WebXDC", file=path_to_webxdc)
+    ac2_webxdc_msg = ac2.wait_for_incoming_msg()
+    assert ac2_webxdc_msg.get_snapshot().text == "WebXDC"
+
+    ac1_ac2_chat.send_text("Hello!")
+    ac2_hello_msg = ac2.wait_for_incoming_msg()
+    ac2_hello_msg_snapshot = ac2_hello_msg.get_snapshot()
+    assert ac2_hello_msg_snapshot.text == "Hello!"
+    ac2_hello_msg_snapshot.chat.accept()
+
+    ac2_webxdc_msg.send_webxdc_realtime_advertisement()
+    while 1:
+        event = ac1.wait_for_event()
+        if event.kind == EventType.WEBXDC_REALTIME_ADVERTISEMENT_RECEIVED:
+            assert event.msg_id == ac1_webxdc_msg.id
+            break

deltachat-rpc-client/tests/test_securejoin.py

@@ -73,22 +73,25 @@ def test_qr_securejoin(acfactory, protect, tmp_path):
     qr_code = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
 
-    # Check that at least some of the handshake messages are deleted.
+    # Alice deletes "vg-request".
+    while True:
+        event = alice.wait_for_event()
+        if event["kind"] == "ImapMessageDeleted":
+            break
+    alice.wait_for_securejoin_inviter_success()
+    # Bob deletes "vg-auth-required", Alice deletes "vg-request-with-auth".
     for ac in [alice, bob]:
         while True:
             event = ac.wait_for_event()
             if event["kind"] == "ImapMessageDeleted":
                 break
-
+    bob.wait_for_securejoin_joiner_success()
-    alice.wait_for_securejoin_inviter_success()
 
     # Test that Alice verified Bob's profile.
     alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
     alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
     assert alice_contact_bob_snapshot.is_verified
 
-    bob.wait_for_securejoin_joiner_success()
-
     snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
     assert snapshot.chat.get_basic_snapshot().is_protected == protect

deltachat-rpc-client/tests/test_something.py

@@ -12,7 +12,6 @@ import pytest
 
 from deltachat_rpc_client import Contact, EventType, Message, events
 from deltachat_rpc_client.const import DownloadState, MessageState
-from deltachat_rpc_client.direct_imap import DirectImap
 from deltachat_rpc_client.rpc import JsonRpcError
 
 
@@ -57,8 +56,8 @@ def test_acfactory(acfactory) -> None:
             if event.progress == 1000:  # Success
                 break
         else:
-            print(event)
+            logging.info(event)
-    print("Successful configuration")
+    logging.info("Successful configuration")
 
 
 def test_configure_starttls(acfactory) -> None:
@@ -246,6 +245,7 @@ def test_contact(acfactory) -> None:
     assert repr(alice_contact_bob)
     alice_contact_bob.block()
     alice_contact_bob.unblock()
+    alice_contact_bob.reset_encryption()
     alice_contact_bob.set_name("new name")
     alice_contact_bob.get_encryption_info()
     snapshot = alice_contact_bob.get_snapshot()
@@ -535,7 +535,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
     assert list(reactions.reactions_by_contact.values())[0] == [react_str]
 
 
-def test_reactions_for_a_reordering_move(acfactory):
+def test_reactions_for_a_reordering_move(acfactory, direct_imap):
     """When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
     their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
     processed by receive_imf in the wrong order, and, particularly, reactions were processed before
@@ -559,7 +559,7 @@ def test_reactions_for_a_reordering_move(acfactory):
     msg1.send_reaction(react_str).wait_until_delivered()
 
     logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
-    ac2_direct_imap = DirectImap(ac2)
+    ac2_direct_imap = direct_imap(ac2)
     ac2_direct_imap.connect()
     for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
         ac2_direct_imap.conn.move(uid, "DeltaChat")

deltachat-rpc-client/tests/test_webxdc.py

@@ -24,6 +24,9 @@ def test_webxdc(acfactory) -> None:
         "name": "Chess Board",
         "sourceCodeUrl": None,
         "summary": None,
+        "selfAddr": webxdc_info["selfAddr"],
+        "sendUpdateInterval": 1000,
+        "sendUpdateMaxSize": 18874368,
     }
 
     status_updates = message.get_webxdc_status_updates()

deltachat-rpc-client/tox.ini

@@ -16,6 +16,7 @@ deps =
     pytest
     pytest-timeout
     pytest-xdist
+    imap-tools
 
 [testenv:lint]
 skipsdist = True

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.147.1"
+version = "1.152.2"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deltachat-rpc-server/npm-package/README.md

@@ -65,13 +65,13 @@ so by default it uses the prebuilds.
   - this will run `update_optional_dependencies_and_version.js` (in the `prepack` script),
     which puts all platform packages into `optionalDependencies` and updates the `version` in `package.json`
 
-## How to build a version you can use localy on your host machine for development
+## How to build a version you can use locally on your host machine for development
 
-You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have seperate scripts for making it work for local installation.
+You can not install the npm packet from the previous section locally, unless you have a local npm registry set up where you upload it too. This is why we have separate scripts for making it work for local installation.
 
 - If you just need your host platform run `python scripts/make_local_dev_version.py`
 - note: this clears the `platform_package` folder
-- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple plaftorms with `build_platform_package.py`
+- (advanced) If you need more than one platform for local install you can just run `node scripts/update_optional_dependencies_and_version.js` after building multiple platforms with `build_platform_package.py`
 
 ## Thanks to nlnet
 

deltachat-rpc-server/npm-package/package.json

@@ -15,5 +15,5 @@
   },
   "type": "module",
   "types": "index.d.ts",
-  "version": "1.147.1"
+  "version": "1.152.2"
 }

deltachat-rpc-server/npm-package/scripts/update_optional_dependencies_and_version.js

@@ -6,7 +6,7 @@ const expected_cwd = join(dirname(fileURLToPath(import.meta.url)), "..");
 
 if (process.cwd() !== expected_cwd) {
   console.error(
-    "CWD missmatch: this script needs to be run from " + expected_cwd,
+    "CWD mismatch: this script needs to be run from " + expected_cwd,
     { actual: process.cwd(), expected: expected_cwd }
   );
   process.exit(1);
@@ -40,7 +40,7 @@ const platform_package_names = await Promise.all(
         "has a different version than the version of the rpc server.",
         { rpc_server: version, platform_package: p.version }
       );
-      throw new Error("version missmatch");
+      throw new Error("version mismatch");
     }
     return { folder_name: name, package_name: p.name };
   })

deltachat-rpc-server/src/main.rs

@@ -66,7 +66,7 @@ async fn main_impl() -> Result<()> {
 
     // Logs from `log` crate and traces from `tracing` crate
     // are configurable with `RUST_LOG` environment variable
-    // and go to stderr to avoid interferring with JSON-RPC using stdout.
+    // and go to stderr to avoid interfering with JSON-RPC using stdout.
     tracing_subscriber::fmt()
         .with_env_filter(EnvFilter::from_default_env())
         .with_writer(std::io::stderr)

deny.toml

@@ -11,6 +11,16 @@ ignore = [
 
     # Unmaintained encoding
     "RUSTSEC-2021-0153",
+
+    # Unmaintained proc-macro-error
+    # <https://rustsec.org/advisories/RUSTSEC-2024-0370>
+    "RUSTSEC-2024-0370",
+
+    # Unmaintained instant
+    "RUSTSEC-2024-0384",
+
+    # idna 0.5.0
+    "RUSTSEC-2024-0421",
 ]
 
 [bans]
@@ -26,12 +36,11 @@ skip = [
      { name = "event-listener", version = "2.5.3" },
      { name = "event-listener", version = "4.0.3" },
      { name = "fastrand", version = "1.9.0" },
+     { name = "fiat-crypto", version = "0.1.20" },
      { name = "futures-lite", version = "1.13.0" },
      { name = "getrandom", version = "<0.2" },
-     { name = "h2", version = "0.3.26" },
-     { name = "http-body", version = "0.4.6" },
      { name = "http", version = "0.2.12" },
-     { name = "hyper", version = "0.14.28" },
+     { name = "idna", version = "0.5.0" },
      { name = "nix", version = "0.26.4" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
@@ -71,6 +80,7 @@ allow = [
     "MIT",
     "MPL-2.0",
     "OpenSSL",
+    "Unicode-3.0",
     "Unicode-DFS-2016",
     "Zlib",
 ]

flake.lock

@@ -7,11 +7,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1729369131,
+        "lastModified": 1731356359,
-        "narHash": "sha256-PtfScp+nQd1PsT5rf0Qgjdbsh4Iag6R1ivYMWLizyIc=",
+        "narHash": "sha256-vYqJnu6jotmWpPT4DgzHVdvNIZcKZCIUqS8QaptsZA0=",
         "owner": "tadfisher",
         "repo": "android-nixpkgs",
-        "rev": "82bffbf3f06bdccf44fc62a9bd4f152ac80a55b0",
+        "rev": "c028ead7e88edb2e94cd7c90ee37593f63ae494a",
         "type": "github"
       },
       "original": {
@@ -47,16 +47,17 @@
         "rust-analyzer-src": "rust-analyzer-src"
       },
       "locked": {
-        "lastModified": 1729375822,
+        "lastModified": 1711088506,
-        "narHash": "sha256-bRo4xVwUhvJ4Gz+OhWMREFMdBOYSw4Yi1Apj01ebbug=",
+        "narHash": "sha256-USdlY7Tx2oJWqFBpp10+03+h7eVhpkQ4s9t1ERjeIJE=",
         "owner": "nix-community",
         "repo": "fenix",
-        "rev": "2853e7d9b5c52a148a9fb824bfe4f9f433f557ab",
+        "rev": "85f4139f3c092cf4afd9f9906d7ed218ef262c97",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
         "repo": "fenix",
+        "rev": "85f4139f3c092cf4afd9f9906d7ed218ef262c97",
         "type": "github"
       }
     },
@@ -114,13 +115,32 @@
         "type": "github"
       }
     },
+    "new-fenix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs_4",
+        "rust-analyzer-src": "rust-analyzer-src_2"
+      },
+      "locked": {
+        "lastModified": 1734417396,
+        "narHash": "sha256-32x1Z+Pz3Jv0cK9EG56cFTKXy/mZ/c+Ikxw+aVfKHp4=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "a18d41b26e998e95a598858fdb86ba22fb5da47d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "nix-filter": {
       "locked": {
-        "lastModified": 1710156097,
+        "lastModified": 1730207686,
-        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "narHash": "sha256-SCHiL+1f7q9TAnxpasriP6fMarWE5H43t25F5/9e28I=",
         "owner": "numtide",
         "repo": "nix-filter",
-        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "rev": "776e68c1d014c3adde193a18db9d738458cd2ba4",
         "type": "github"
       },
       "original": {
@@ -131,11 +151,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1729256560,
+        "lastModified": 1731139594,
-        "narHash": "sha256-/uilDXvCIEs3C9l73JTACm4quuHUsIHcns1c+cHUJwA=",
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4c2fcb090b1f3e5b47eaa7bd33913b574a11e0a0",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -147,11 +167,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1729070438,
+        "lastModified": 1731139594,
-        "narHash": "sha256-KOTTUfPkugH52avUvXGxvWy8ibKKj4genodIYUED+Kc=",
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "5785b6bb5eaae44e627d541023034e1601455827",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -163,12 +183,10 @@
     },
     "nixpkgs_3": {
       "locked": {
-        "lastModified": 1729265718,
+        "lastModified": 0,
-        "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=",
+        "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=",
-        "owner": "NixOS",
+        "path": "/nix/store/zq2axpgzd5kykk1v446rkffj3bxa2m2h-source",
-        "repo": "nixpkgs",
+        "type": "path"
-        "rev": "ccc0c2126893dd20963580b6478d1a10a4512185",
-        "type": "github"
       },
       "original": {
         "id": "nixpkgs",
@@ -177,11 +195,27 @@
     },
     "nixpkgs_4": {
       "locked": {
-        "lastModified": 1729256560,
+        "lastModified": 1734119587,
-        "narHash": "sha256-/uilDXvCIEs3C9l73JTACm4quuHUsIHcns1c+cHUJwA=",
+        "narHash": "sha256-AKU6qqskl0yf2+JdRdD0cfxX4b9x3KKV5RqA6wijmPM=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "4c2fcb090b1f3e5b47eaa7bd33913b574a11e0a0",
+        "rev": "3566ab7246670a43abd2ffa913cc62dad9cdf7d5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_5": {
+      "locked": {
+        "lastModified": 1731139594,
+        "narHash": "sha256-IigrKK3vYRpUu+HEjPL/phrfh7Ox881er1UEsZvw9Q4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "76612b17c0ce71689921ca12d9ffdc9c23ce40b2",
         "type": "github"
       },
       "original": {
@@ -197,18 +231,36 @@
         "fenix": "fenix",
         "flake-utils": "flake-utils_2",
         "naersk": "naersk",
+        "new-fenix": "new-fenix",
         "nix-filter": "nix-filter",
-        "nixpkgs": "nixpkgs_4"
+        "nixpkgs": "nixpkgs_5"
       }
     },
     "rust-analyzer-src": {
       "flake": false,
       "locked": {
-        "lastModified": 1729255720,
+        "lastModified": 1731342671,
-        "narHash": "sha256-yODOuZxBkS0UfqMa6nmbqNbVfIbsu0tYLbV5vZzmsqI=",
+        "narHash": "sha256-36eYDHoPzjavnpmEpc2MXdzMk557S0YooGms07mDuKk=",
         "owner": "rust-lang",
         "repo": "rust-analyzer",
-        "rev": "72b214fbfbe6f7b95a7877b962783bd42062cc0a",
+        "rev": "fc98e0657abf3ce07eed513e38274c89bbb2f8ad",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "rust-analyzer-src_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1734386068,
+        "narHash": "sha256-Py025JiD9lcPmldB7X1AEjq3WBTS60jZUJRtTDonmaE=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "0a706f7d2ac093985eae317781200689cfd48b78",
         "type": "github"
       },
       "original": {

flake.nix

@@ -1,14 +1,19 @@
 {
   description = "Delta Chat core";
   inputs = {
-    fenix.url = "github:nix-community/fenix";
+    # Old Rust to build releases.
+    fenix.url = "github:nix-community/fenix?rev=85f4139f3c092cf4afd9f9906d7ed218ef262c97";
+
+    # New Rust for development shell.
+    new-fenix.url = "github:nix-community/fenix";
+
     flake-utils.url = "github:numtide/flake-utils";
     naersk.url = "github:nix-community/naersk";
     nix-filter.url = "github:numtide/nix-filter";
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
     android.url = "github:tadfisher/android-nixpkgs";
   };
-  outputs = { self, nixpkgs, flake-utils, nix-filter, naersk, fenix, android }:
+  outputs = { self, nixpkgs, flake-utils, nix-filter, naersk, fenix, new-fenix, android }:
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
@@ -18,9 +23,9 @@
         manifest = (pkgs.lib.importTOML ./Cargo.toml).package;
         androidSdk = android.sdk.${system} (sdkPkgs:
           builtins.attrValues {
-            inherit (sdkPkgs) ndk-24-0-8215888 cmdline-tools-latest;
+            inherit (sdkPkgs) ndk-27-0-11902837 cmdline-tools-latest;
           });
-        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/24.0.8215888";
+        androidNdkRoot = "${androidSdk}/share/android-sdk/ndk/27.0.11902837";
 
         rustSrc = nix-filter.lib {
           root = ./.;
@@ -257,13 +262,21 @@
 
         androidAttrs = {
           armeabi-v7a = {
-            cc = "armv7a-linux-androideabi19-clang";
+            cc = "armv7a-linux-androideabi21-clang";
             rustTarget = "armv7-linux-androideabi";
           };
           arm64-v8a = {
             cc = "aarch64-linux-android21-clang";
             rustTarget = "aarch64-linux-android";
           };
+          x86 = {
+            cc = "i686-linux-android21-clang";
+            rustTarget = "i686-linux-android";
+          };
+          x86_64 = {
+            cc = "x86_64-linux-android21-clang";
+            rustTarget = "x86_64-linux-android";
+          };
         };
 
         mkAndroidRustPackage = arch: packageName:
@@ -355,6 +368,8 @@
           mkRustPackages "x86_64-linux" //
           mkRustPackages "armv7l-linux" //
           mkRustPackages "armv6l-linux" //
+          mkRustPackages "x86_64-darwin" //
+          mkRustPackages "aarch64-darwin" //
           mkAndroidPackages "armeabi-v7a" //
           mkAndroidPackages "arm64-v8a" //
           mkAndroidPackages "x86" //
@@ -471,8 +486,8 @@
                   pkgs.python3
                   pkgs.python3Packages.wheel
                 ];
-                buildPhase = ''python3 scripts/wheel-rpc-server.py source deltachat-rpc-server-${manifest.version}.tar.gz'';
+                buildPhase = ''python3 scripts/wheel-rpc-server.py source deltachat_rpc_server-${manifest.version}.tar.gz'';
-                installPhase = ''mkdir -p $out; cp -av deltachat-rpc-server-${manifest.version}.tar.gz $out'';
+                installPhase = ''mkdir -p $out; cp -av deltachat_rpc_server-${manifest.version}.tar.gz $out'';
               };
 
             deltachat-rpc-client =
@@ -525,15 +540,17 @@
               };
           };
 
-        devShells.default = let 
+        devShells.default =
+          let
             pkgs = import nixpkgs {
               system = system;
-            overlays = [ fenix.overlays.default ];
+              overlays = [ new-fenix.overlays.default ];
             };
-          in pkgs.mkShell {
+          in
+          pkgs.mkShell {
 
             buildInputs = with pkgs; [
-            (fenix.packages.${system}.complete.withComponents [
+              (new-fenix.packages.${system}.complete.withComponents [
                 "cargo"
                 "clippy"
                 "rust-src"

format-flowed/src/lib.rs

@@ -8,6 +8,8 @@
 //! is assumed to be set to "no".
 //!
 //! For received messages, DelSp parameter is honoured.
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///

node/constants.js

@@ -1,6 +1,7 @@
 // Generated!
 
 module.exports = {
+  DC_CERTCK_ACCEPT_INVALID: 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES: 3,
   DC_CERTCK_AUTO: 0,
   DC_CERTCK_STRICT: 1,
@@ -30,6 +31,8 @@ module.exports = {
   DC_DOWNLOAD_IN_PROGRESS: 1000,
   DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE: 2200,
+  DC_EVENT_ACCOUNTS_CHANGED: 2302,
+  DC_EVENT_ACCOUNTS_ITEM_CHANGED: 2303,
   DC_EVENT_CHANNEL_OVERFLOW: 2400,
   DC_EVENT_CHATLIST_CHANGED: 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED: 2301,
@@ -51,6 +54,7 @@ module.exports = {
   DC_EVENT_INCOMING_MSG: 2005,
   DC_EVENT_INCOMING_MSG_BUNCH: 2006,
   DC_EVENT_INCOMING_REACTION: 2002,
+  DC_EVENT_INCOMING_WEBXDC_NOTIFY: 2003,
   DC_EVENT_INFO: 100,
   DC_EVENT_LOCATION_CHANGED: 2035,
   DC_EVENT_MSGS_CHANGED: 2000,

node/events.js

@@ -17,6 +17,7 @@ module.exports = {
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
   2002: 'DC_EVENT_INCOMING_REACTION',
+  2003: 'DC_EVENT_INCOMING_WEBXDC_NOTIFY',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -43,5 +44,7 @@ module.exports = {
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2302: 'DC_EVENT_ACCOUNTS_CHANGED',
+  2303: 'DC_EVENT_ACCOUNTS_ITEM_CHANGED',
   2400: 'DC_EVENT_CHANNEL_OVERFLOW'
 }

node/lib/constants.ts

@@ -1,6 +1,7 @@
 // Generated!
 
 export enum C {
+  DC_CERTCK_ACCEPT_INVALID = 2,
   DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3,
   DC_CERTCK_AUTO = 0,
   DC_CERTCK_STRICT = 1,
@@ -30,6 +31,8 @@ export enum C {
   DC_DOWNLOAD_IN_PROGRESS = 1000,
   DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE = 2200,
+  DC_EVENT_ACCOUNTS_CHANGED = 2302,
+  DC_EVENT_ACCOUNTS_ITEM_CHANGED = 2303,
   DC_EVENT_CHANNEL_OVERFLOW = 2400,
   DC_EVENT_CHATLIST_CHANGED = 2300,
   DC_EVENT_CHATLIST_ITEM_CHANGED = 2301,
@@ -51,6 +54,7 @@ export enum C {
   DC_EVENT_INCOMING_MSG = 2005,
   DC_EVENT_INCOMING_MSG_BUNCH = 2006,
   DC_EVENT_INCOMING_REACTION = 2002,
+  DC_EVENT_INCOMING_WEBXDC_NOTIFY = 2003,
   DC_EVENT_INFO = 100,
   DC_EVENT_LOCATION_CHANGED = 2035,
   DC_EVENT_MSGS_CHANGED = 2000,
@@ -324,6 +328,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2000: 'DC_EVENT_MSGS_CHANGED',
   2001: 'DC_EVENT_REACTIONS_CHANGED',
   2002: 'DC_EVENT_INCOMING_REACTION',
+  2003: 'DC_EVENT_INCOMING_WEBXDC_NOTIFY',
   2005: 'DC_EVENT_INCOMING_MSG',
   2006: 'DC_EVENT_INCOMING_MSG_BUNCH',
   2008: 'DC_EVENT_MSGS_NOTICED',
@@ -350,5 +355,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2200: 'DC_EVENT_ACCOUNTS_BACKGROUND_FETCH_DONE',
   2300: 'DC_EVENT_CHATLIST_CHANGED',
   2301: 'DC_EVENT_CHATLIST_ITEM_CHANGED',
+  2302: 'DC_EVENT_ACCOUNTS_CHANGED',
+  2303: 'DC_EVENT_ACCOUNTS_ITEM_CHANGED',
   2400: 'DC_EVENT_CHANNEL_OVERFLOW',
 }

node/lib/message.ts

@@ -299,10 +299,6 @@ export class Message {
     return Boolean(binding.dcn_msg_is_forwarded(this.dc_msg))
   }
 
-  isIncreation() {
-    return Boolean(binding.dcn_msg_is_increation(this.dc_msg))
-  }
-
   isInfo() {
     return Boolean(binding.dcn_msg_is_info(this.dc_msg))
   }

node/src/module.c

@@ -2374,17 +2374,6 @@ NAPI_METHOD(dcn_msg_is_forwarded) {
   NAPI_RETURN_INT32(is_forwarded);
 }
 
-NAPI_METHOD(dcn_msg_is_increation) {
-  NAPI_ARGV(1);
-  NAPI_DC_MSG();
-
-  //TRACE("calling..");
-  int is_increation = dc_msg_is_increation(dc_msg);
-  //TRACE("result %d", is_increation);
-
-  NAPI_RETURN_INT32(is_increation);
-}
-
 NAPI_METHOD(dcn_msg_is_info) {
   NAPI_ARGV(1);
   NAPI_DC_MSG();
@@ -3555,7 +3544,6 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_msg_has_location);
   NAPI_EXPORT_FUNCTION(dcn_msg_has_html);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_forwarded);
-  NAPI_EXPORT_FUNCTION(dcn_msg_is_increation);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_info);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_sent);
   NAPI_EXPORT_FUNCTION(dcn_msg_is_setupmessage);

node/test/test.mjs

@@ -536,7 +536,6 @@ describe('Offline Tests with unconfigured account', function () {
     strictEqual(msg.getWidth(), 0, 'no message width')
     strictEqual(msg.isDeadDrop(), false, 'not deaddrop')
     strictEqual(msg.isForwarded(), false, 'not forwarded')
-    strictEqual(msg.isIncreation(), false, 'not in creation')
     strictEqual(msg.isInfo(), false, 'not an info message')
     strictEqual(msg.isSent(), false, 'messge is not sent')
     strictEqual(msg.isSetupmessage(), false, 'not an autocrypt setup message')

package.json

@@ -55,5 +55,5 @@
     "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.147.1"
+  "version": "1.152.2"
 }

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "deltachat"
-version = "1.147.1"
+version = "1.152.2"
 description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
 readme = "README.rst"
 requires-python = ">=3.7"

python/src/deltachat/account.py

@@ -671,9 +671,6 @@ class Account:
     def get_connectivity_html(self) -> str:
         return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
 
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """start this account's IO scheduling (Rust-core async scheduler).
 

python/src/deltachat/chat.py

@@ -271,8 +271,7 @@ class Chat:
 
         :param msg: a :class:`deltachat.message.Message` instance
            previously returned by
-           e.g. :meth:`deltachat.message.Message.new_empty` or
+           e.g. :meth:`deltachat.message.Message.new_empty`.
-           :meth:`prepare_file`.
         :raises ValueError: if message can not be sent.
 
         :returns: a :class:`deltachat.message.Message` instance as
@@ -341,37 +340,6 @@ class Chat:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
 
-    def prepare_message(self, msg):
-        """prepare a message for sending.
-
-        :param msg: the message to be prepared.
-        :returns: a :class:`deltachat.message.Message` instance.
-           This is the same object that was passed in, which
-           has been modified with the new state of the core.
-        """
-        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
-        if msg_id == 0:
-            raise ValueError("message could not be prepared")
-        # modify message in place to avoid bad state for the caller
-        msg._dc_msg = Message.from_db(self.account, msg_id)._dc_msg
-        return msg
-
-    def prepare_message_file(self, path, mime_type=None, view_type="file"):
-        """prepare a message for sending and return the resulting Message instance.
-
-        To actually send the message, call :meth:`send_prepared`.
-        The file must be inside the blob directory.
-
-        :param path: path to the file.
-        :param mime_type: the mime-type of this file, defaults to auto-detection.
-        :param view_type: "text", "image", "gif", "audio", "video", "file"
-        :raises ValueError: if message can not be prepared/chat does not exist.
-        :returns: the resulting :class:`Message` instance
-        """
-        msg = Message.new_empty(self.account, view_type)
-        msg.set_file(path, mime_type)
-        return self.prepare_message(msg)
-
     def send_prepared(self, message):
         """send a previously prepared message.
 

python/src/deltachat/events.py

@@ -158,12 +158,6 @@ class FFIEventTracker:
 
             self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
 
-    def wait_for_all_work_done(self):
-        while True:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile(f"(?:{event_name_regex}).*")

python/src/deltachat/testplugin.py

@@ -152,7 +152,7 @@ class TestProcess:
 
     def get_liveconfig_producer(self):
         """provide live account configs, cached on a per-test-process scope
-        so that test functions can re-use already known live configs.
+        so that test functions can reuse already known live configs.
         """
         chatmail_opt = self.pytestconfig.getoption("--chatmail")
         if chatmail_opt:

python/tests/test_1_online.py

@@ -1366,10 +1366,9 @@ def test_quote_encrypted(acfactory, lp):
         msg_draft.quote = quoted_msg
         chat.set_draft(msg_draft)
 
-        # Get the draft, prepare and send it.
+        # Get the draft and send it.
         msg_draft = chat.get_draft()
-        msg_out = chat.prepare_message(msg_draft)
+        chat.send_msg(msg_draft)
-        chat.send_prepared(msg_out)
 
         chat.set_draft(None)
         assert chat.get_draft() is None
@@ -1899,10 +1898,11 @@ def test_connectivity(acfactory, lp):
 
     ac1.start_io()
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_CONNECTING)
-    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_CONNECTING, dc.const.DC_CONNECTIVITY_CONNECTED)
+    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_CONNECTING, dc.const.DC_CONNECTIVITY_WORKING)
+    ac1._evtracker.wait_for_connectivity_change(dc.const.DC_CONNECTIVITY_WORKING, dc.const.DC_CONNECTIVITY_CONNECTED)
 
     lp.sec(
-        "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, "
+        "Test that after calling start_io(), maybe_network() and waiting for `DC_CONNECTIVITY_CONNECTED`, "
         "all messages are fetched",
     )
 
@@ -1911,7 +1911,7 @@ def test_connectivity(acfactory, lp):
         ac2.create_chat(ac1).send_text("Hi")
         idle1.wait_for_new_message()
     ac1.maybe_network()
-    ac1._evtracker.wait_for_all_work_done()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_CONNECTED)
     msgs = ac1.create_chat(ac2).get_messages()
     assert len(msgs) == 1
     assert msgs[0].text == "Hi"
@@ -1927,30 +1927,6 @@ def test_connectivity(acfactory, lp):
     assert len(msgs) == 2
     assert msgs[1].text == "Hi 2"
 
-    lp.sec("Test that the connectivity doesn't flicker to WORKING if there are no new messages")
-
-    ac1.maybe_network()
-    while 1:
-        assert ac1.get_connectivity() == dc.const.DC_CONNECTIVITY_CONNECTED
-        if ac1.all_work_done():
-            break
-        ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    lp.sec("Test that the connectivity doesn't flicker to WORKING if the sender of the message is blocked")
-    ac1.create_contact(ac2).block()
-
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        ac2.create_chat(ac1).send_text("Hi")
-        idle1.wait_for_new_message()
-    ac1.maybe_network()
-
-    while 1:
-        assert ac1.get_connectivity() == dc.const.DC_CONNECTIVITY_CONNECTED
-        if ac1.all_work_done():
-            break
-        ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     lp.sec("Test that the connectivity is NOT_CONNECTED if the password is wrong")
 
     ac1.set_config("configured_mail_pw", "abc")
@@ -1961,32 +1937,6 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
-def test_all_work_done(acfactory, lp):
-    """
-    Tests that calling start_io() immediately followed by maybe_network()
-    and then waiting for all_work_done() reliably fetches the messages
-    delivered while account was offline.
-    In other words, connectivity should not change to a state
-    where all_work_done() returns true until IMAP connection goes idle.
-    """
-    ac1, ac2 = acfactory.get_online_accounts(2)
-
-    ac1.stop_io()
-    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
-
-    ac1.direct_imap.select_config_folder("inbox")
-    with ac1.direct_imap.idle() as idle1:
-        ac2.create_chat(ac1).send_text("Hi")
-        idle1.wait_for_new_message()
-
-    ac1.start_io()
-    ac1.maybe_network()
-    ac1._evtracker.wait_for_all_work_done()
-    msgs = ac1.create_chat(ac2).get_messages()
-    assert len(msgs) == 1
-    assert msgs[0].text == "Hi"
-
-
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.
@@ -2209,6 +2159,19 @@ def test_configure_error_msgs_wrong_pw(acfactory):
     # Password is wrong so it definitely has to say something about "password"
     assert "password" in ev.data2
 
+    ac1.stop_io()
+    ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+    ac1.configure()
+    while True:
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        print(f"Configuration progress: {ev.data1}")
+        if ev.data1 == 0:
+            break
+    assert "password" in ev.data2
+    # Account will continue to work with the old password, so if it becomes wrong, a notification
+    # must be shown.
+    assert ac1.get_config("notify_about_wrong_pw") == "1"
+
 
 def test_configure_error_msgs_invalid_server(acfactory):
     ac2 = acfactory.get_unconfigured_account()
@@ -2327,9 +2290,8 @@ def test_group_quote(acfactory, lp):
     reply_msg = Message.new_empty(msg.chat.account, "text")
     reply_msg.set_text("reply")
     reply_msg.quote = msg
-    reply_msg = msg.chat.prepare_message(reply_msg)
     assert reply_msg.quoted_text == "hello"
-    msg.chat.send_prepared(reply_msg)
+    msg.chat.send_msg(reply_msg)
 
     lp.sec("ac3: receiving reply")
     received_reply = ac3._evtracker.wait_next_incoming_message()

python/tests/test_2_increation.py

@@ -1,107 +0,0 @@
-import os.path
-import shutil
-from filecmp import cmp
-
-import pytest
-
-
-def wait_msg_delivered(account, msg_list):
-    """wait for one or more MSG_DELIVERED events to match msg_list contents."""
-    msg_list = list(msg_list)
-    while msg_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        msg_list.remove((ev.data1, ev.data2))
-
-
-def wait_msgs_changed(account, msgs_list):
-    """wait for one or more MSGS_CHANGED events to match msgs_list contents."""
-    account.log(f"waiting for msgs_list={msgs_list}")
-    msgs_list = list(msgs_list)
-    while msgs_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        for i, (data1, data2) in enumerate(msgs_list):
-            if ev.data1 == data1:
-                if data2 is None or ev.data2 == data2:
-                    del msgs_list[i]
-                    break
-        else:
-            account.log(f"waiting mismatch data1={data1} data2={data2}")
-    return ev.data2
-
-
-class TestOnlineInCreation:
-    def test_increation_not_blobdir(self, tmp_path, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating in-creation file outside of blobdir")
-        assert str(tmp_path) != ac1.get_blobdir()
-        src = tmp_path / "file.txt"
-        src.touch()
-        with pytest.raises(Exception):
-            chat.prepare_message_file(str(src))
-
-    def test_no_increation_copies_to_blobdir(self, tmp_path, acfactory, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-        chat = ac1.create_chat(ac2)
-
-        lp.sec("Creating file outside of blobdir")
-        assert str(tmp_path) != ac1.get_blobdir()
-        src = tmp_path / "file.txt"
-        src.write_text("hello there\n")
-        msg = chat.send_file(str(src))
-        assert msg.filename.startswith(os.path.join(ac1.get_blobdir(), "file"))
-        assert msg.filename.endswith(".txt")
-
-    def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_online_accounts(2)
-
-        chat = ac1.create_chat(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
-
-        lp.sec("create a message with a file in creation")
-        orig = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), "d.png")
-        with open(path, "x") as fp:
-            fp.write("preparing")
-        prepared_original = chat.prepare_message_file(path)
-        assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
-
-        lp.sec("create a new group")
-        chat2 = ac1.create_group_chat("newgroup")
-        wait_msgs_changed(ac1, [(0, 0)])
-
-        lp.sec("add a contact to new group")
-        chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(chat2.id, None)])
-
-        lp.sec("forward the message while still in creation")
-        ac1.forward_messages([prepared_original], chat2)
-        forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
-        forwarded_msg = ac1.get_message_by_id(forwarded_id)
-        assert forwarded_msg.is_out_preparing()
-
-        lp.sec("finish creating the file and send it")
-        assert prepared_original.is_out_preparing()
-        shutil.copyfile(orig, path)
-        chat.send_prepared(prepared_original)
-        assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
-
-        lp.sec("check that both forwarded and original message are proper.")
-        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
-        fwd_msg = ac1.get_message_by_id(forwarded_id)
-        assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
-
-        lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
-        lp.sec("wait1 for original or forwarded messages to arrive")
-        received_original = ac2._evtracker.wait_next_incoming_message()
-        assert cmp(received_original.filename, orig, shallow=False)
-
-        lp.sec("wait2 for original or forwarded messages to arrive")
-        received_copy = ac2._evtracker.wait_next_incoming_message()
-        assert received_copy.id != received_original.id
-        assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_3_offline.py

@@ -378,30 +378,6 @@ class TestOfflineChat:
         with pytest.raises(ValueError):
             chat1.send_text("msg1")
 
-    def test_prepare_message_and_send(self, ac1, chat1):
-        msg = chat1.prepare_message(Message.new_empty(chat1.account, "text"))
-        msg.set_text("hello world")
-        assert msg.text == "hello world"
-        assert msg.id > 0
-        chat1.send_prepared(msg)
-        assert "Sent" in msg.get_message_info()
-        str(msg)
-        repr(msg)
-        assert msg == ac1.get_message_by_id(msg.id)
-
-    def test_prepare_file(self, ac1, chat1):
-        blobdir = ac1.get_blobdir()
-        p = os.path.join(blobdir, "somedata.txt")
-        with open(p, "w") as f:
-            f.write("some data")
-        message = chat1.prepare_message_file(p)
-        assert message.id > 0
-        message.set_text("hello world")
-        assert message.is_out_preparing()
-        assert message.text == "hello world"
-        chat1.send_prepared(message)
-        assert "Sent" in message.get_message_info()
-
     def test_message_eq_contains(self, chat1):
         msg = chat1.send_text("msg1")
         msg2 = None
@@ -691,8 +667,7 @@ class TestOfflineChat:
         assert os.path.exists(messages[1].filename)
 
     def test_set_get_draft(self, chat1):
-        msg = Message.new_empty(chat1.account, "text")
+        msg1 = Message.new_empty(chat1.account, "text")
-        msg1 = chat1.prepare_message(msg)
         msg1.set_text("hello")
         chat1.set_draft(msg1)
         msg1.set_text("obsolete")
@@ -705,27 +680,12 @@ class TestOfflineChat:
         ac1 = acfactory.get_pseudo_configured_account()
         ac2 = acfactory.get_pseudo_configured_account()
         qr = ac1.get_setup_contact_qr()
-        assert qr.startswith("OPENPGP4FPR:")
+        assert qr.startswith("https://i.delta.chat")
         res = ac2.check_qr(qr)
         assert res.is_ask_verifycontact()
         assert not res.is_ask_verifygroup()
         assert res.contact_id == 10
 
-    def test_quote(self, chat1):
-        """Offline quoting test"""
-        msg = Message.new_empty(chat1.account, "text")
-        msg.set_text("Multi\nline\nmessage")
-        assert msg.quoted_text is None
-
-        # Prepare message to assign it a Message-Id.
-        # Messages without Message-Id cannot be quoted.
-        msg = chat1.prepare_message(msg)
-
-        reply_msg = Message.new_empty(chat1.account, "text")
-        reply_msg.set_text("reply")
-        reply_msg.quote = msg
-        assert reply_msg.quoted_text == "Multi\nline\nmessage"
-
     def test_group_chat_many_members_add_remove(self, ac1, lp):
         lp.sec("ac1: creating group chat with 10 other members")
         chat = ac1.create_group_chat(name="title1")

python/tests/test_4_lowlevel.py

@@ -216,7 +216,7 @@ def test_logged_ac_process_ffi_failure(acfactory):
     ac1.add_account_plugin(FailPlugin())
     # cause any event eg contact added/changed
     ac1.create_contact("something@example.org")
-    res = cap.get(timeout=10)
+    res = cap.get(timeout=600)
     assert "ac_process_ffi_event" in res
     assert "ZeroDivisionError" in res
     assert "Traceback" in res

release-date.in

@@ -1 +1 @@
-2024-10-13
+2024-12-24

scripts/coredeps/install-rust.sh

@@ -7,7 +7,7 @@ set -euo pipefail
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.82.0
+RUST_VERSION=1.83.0
 
 ARCH="$(uname -m)"
 test -f "/lib/libc.musl-$ARCH.so.1" && LIBC=musl || LIBC=gnu

scripts/wheel-rpc-server.py

@@ -25,7 +25,7 @@ def build_source_package(version, filename):
 
         def pack(name, contents):
             contents = contents.encode()
-            tar_info = tarfile.TarInfo(f"deltachat-rpc-server-{version}/{name}")
+            tar_info = tarfile.TarInfo(f"deltachat_rpc_server-{version}/{name}")
             tar_info.mode = 0o644
             tar_info.size = len(contents)
             pkg.addfile(tar_info, BytesIO(contents))
@@ -167,7 +167,7 @@ def main():
         cargo_manifest = tomllib.load(fp)
     version = cargo_manifest["package"]["version"]
     if sys.argv[1] == "source":
-        filename = f"deltachat-rpc-server-{version}.tar.gz"
+        filename = f"deltachat_rpc_server-{version}.tar.gz"
         build_source_package(version, filename)
     else:
         arch = sys.argv[1]

src/accounts.rs

@@ -5,7 +5,8 @@ use std::future::Future;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context as _, Result};
-use futures::future::join_all;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
@@ -138,6 +139,7 @@ impl Accounts {
         ctx.open("".to_string()).await?;
 
         self.accounts.insert(account_config.id, ctx);
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(account_config.id)
     }
@@ -155,6 +157,7 @@ impl Accounts {
             .build()
             .await?;
         self.accounts.insert(account_config.id, ctx);
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(account_config.id)
     }
@@ -189,6 +192,7 @@ impl Accounts {
                 .context("failed to remove account data")?;
         }
         self.config.remove_account(id).await?;
+        self.emit_event(EventType::AccountsChanged);
 
         Ok(())
     }
@@ -301,20 +305,48 @@ impl Accounts {
     ///
     /// This is an auxiliary function and not part of public API.
     /// Use [Accounts::background_fetch] instead.
-    async fn background_fetch_without_timeout(&self) {
+    async fn background_fetch_no_timeout(accounts: Vec<Context>, events: Events) {
         async fn background_fetch_and_log_error(account: Context) {
             if let Err(error) = account.background_fetch().await {
                 warn!(account, "{error:#}");
             }
         }
 
-        join_all(
+        events.emit(Event {
-            self.accounts
+            id: 0,
-                .values()
+            typ: EventType::Info(format!(
-                .cloned()
+                "Starting background fetch for {} accounts.",
-                .map(background_fetch_and_log_error),
+                accounts.len()
+            )),
+        });
+        let mut futures_unordered: FuturesUnordered<_> = accounts
+            .into_iter()
+            .map(background_fetch_and_log_error)
+            .collect();
+        while futures_unordered.next().await.is_some() {}
+    }
+
+    /// Auxiliary function for [Accounts::background_fetch].
+    async fn background_fetch_with_timeout(
+        accounts: Vec<Context>,
+        events: Events,
+        timeout: std::time::Duration,
+    ) {
+        if let Err(_err) = tokio::time::timeout(
+            timeout,
+            Self::background_fetch_no_timeout(accounts, events.clone()),
         )
-        .await;
+        .await
+        {
+            events.emit(Event {
+                id: 0,
+                typ: EventType::Warning("Background fetch timed out.".to_string()),
+            });
+        }
+        events.emit(Event {
+            id: 0,
+            typ: EventType::AccountsBackgroundFetchDone,
+        });
     }
 
     /// Performs a background fetch for all accounts in parallel with a timeout.
@@ -322,15 +354,13 @@ impl Accounts {
     /// The `AccountsBackgroundFetchDone` event is emitted at the end,
     /// process all events until you get this one and you can safely return to the background
     /// without forgetting to create notifications caused by timing race conditions.
-    pub async fn background_fetch(&self, timeout: std::time::Duration) {
+    ///
-        if let Err(_err) =
+    /// Returns a future that resolves when background fetch is done,
-            tokio::time::timeout(timeout, self.background_fetch_without_timeout()).await
+    /// but does not capture `&self`.
-        {
+    pub fn background_fetch(&self, timeout: std::time::Duration) -> impl Future<Output = ()> {
-            self.emit_event(EventType::Warning(
+        let accounts: Vec<Context> = self.accounts.values().cloned().collect();
-                "Background fetch timed out.".to_string(),
+        let events = self.events.clone();
-            ));
+        Self::background_fetch_with_timeout(accounts, events, timeout)
-        }
-        self.emit_event(EventType::AccountsBackgroundFetchDone);
     }
 
     /// Emits a single event.
@@ -344,7 +374,7 @@ impl Accounts {
     }
 
     /// Sets notification token for Apple Push Notification service.
-    pub async fn set_push_device_token(&mut self, token: &str) -> Result<()> {
+    pub async fn set_push_device_token(&self, token: &str) -> Result<()> {
         self.push_subscriber.set_device_token(token).await;
         Ok(())
     }

src/authres.rs

@@ -260,7 +260,6 @@ fn parse_authservid_candidates_config(config: &Option<String>) -> BTreeSet<&str>
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
     use tokio::fs;
     use tokio::io::AsyncReadExt;
 
@@ -520,8 +519,13 @@ Authentication-Results: dkim=";
         handle_authres(&t, &mail, "invalid@rom.com").await.unwrap();
     }
 
+    // Test that Autocrypt works with mailing list.
+    //
+    // Previous versions of Delta Chat ignored Autocrypt based on the List-Post header.
+    // This is not needed: comparing of the From address to Autocrypt header address is enough.
+    // If the mailing list is not rewriting the From header, Autocrypt should be applied.
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_autocrypt_in_mailinglist_ignored() -> Result<()> {
+    async fn test_autocrypt_in_mailinglist_not_ignored() -> Result<()> {
         let mut tcm = TestContextManager::new();
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
@@ -533,28 +537,18 @@ Authentication-Results: dkim=";
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         bob.recv_msg(&sent).await;
         let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
-        assert!(peerstate.is_none());
-
-        // Do the same without the mailing list header, this time the peerstate should be accepted
-        let sent = alice
-            .send_text(alice_bob_chat.id, "hellooo without mailing list")
-            .await;
-        bob.recv_msg(&sent).await;
-        let peerstate = Peerstate::from_addr(&bob, "alice@example.org").await?;
         assert!(peerstate.is_some());
 
-        // This also means that Bob can now write encrypted to Alice:
+        // Bob can now write encrypted to Alice:
         let mut sent = bob
             .send_text(bob_alice_chat.id, "hellooo in the mailinglist again")
             .await;
         assert!(sent.load_from_db().await.get_showpadlock());
 
-        // But if Bob writes to a mailing list, Alice doesn't show a padlock
-        // since she can't verify the signature without accepting Bob's key:
         sent.payload
             .insert_str(0, "List-Post: <mailto:deltachat-community.example.net>\n");
         let rcvd = alice.recv_msg(&sent).await;
-        assert!(!rcvd.get_showpadlock());
+        assert!(rcvd.get_showpadlock());
         assert_eq!(&rcvd.text, "hellooo in the mailinglist again");
 
         Ok(())

src/blob.rs

@@ -279,7 +279,9 @@ impl<'a> BlobObject<'a> {
         let ext: String = name
             .chars()
             .rev()
-            .take_while(|c| !c.is_whitespace())
+            .take_while(|c| {
+                (!c.is_ascii_punctuation() || *c == '.') && !c.is_whitespace() && !c.is_control()
+            })
             .take(33)
             .collect::<Vec<_>>()
             .iter()
@@ -763,7 +765,6 @@ mod tests {
     use fs::File;
 
     use super::*;
-    use crate::chat::{self, create_group_chat, ProtectionStatus};
     use crate::message::{Message, Viewtype};
     use crate::test_utils::{self, TestContext};
 
@@ -983,6 +984,10 @@ mod tests {
         let (stem, ext) = BlobObject::sanitise_name("a. tar.tar.gz");
         assert_eq!(stem, "a. tar");
         assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name("Guia_uso_GNB (v0.8).pdf");
+        assert_eq!(stem, "Guia_uso_GNB (v0.8)");
+        assert_eq!(ext, ".pdf");
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1458,36 +1463,21 @@ mod tests {
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_increation_in_blobdir() -> Result<()> {
+    async fn test_send_gif_as_sticker() -> Result<()> {
-        let t = TestContext::new_alice().await;
+        let bytes = include_bytes!("../test-data/image/image100x50.gif");
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
+        let alice = &TestContext::new_alice().await;
-
+        let file = alice.get_blobdir().join("file").with_extension("gif");
-        let file = t.get_blobdir().join("anyfile.dat");
+        fs::write(&file, &bytes)
-        fs::write(&file, b"bla").await?;
+            .await
-        let mut msg = Message::new(Viewtype::File);
+            .context("failed to write file")?;
+        let mut msg = Message::new(Viewtype::Sticker);
         msg.set_file(file.to_str().unwrap(), None);
-        let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
+        let chat = alice.get_self_chat().await;
-        assert_eq!(prepared_id, msg.id);
+        let sent = alice.send_msg(chat.id, &mut msg).await;
-        assert!(msg.is_increation());
+        let msg = Message::load_from_db(alice, sent.sender_msg_id).await?;
-
+        // Message::force_sticker() wasn't used, still Viewtype::Sticker is preserved because of the
-        let msg = Message::load_from_db(&t, prepared_id).await?;
+        // extension.
-        assert!(msg.is_increation());
+        assert_eq!(msg.get_viewtype(), Viewtype::Sticker);
-
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_increation_not_blobdir() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
-        assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
-
-        let file = t.dir.path().join("anyfile.dat");
-        fs::write(&file, b"bla").await?;
-        let mut msg = Message::new(Viewtype::File);
-        msg.set_file(file.to_str().unwrap(), None);
-        assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
-
         Ok(())
     }
 }

src/chatlist.rs

@@ -394,25 +394,32 @@ impl Chatlist {
             &chat_loaded
         };
 
-        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
+        let lastmsg = if let Some(lastmsg_id) = lastmsg_id {
-            let lastmsg = Message::load_from_db(context, lastmsg_id)
+            // Message may be deleted by the time we try to load it,
+            // so use `load_from_db_optional` instead of `load_from_db`.
+            Message::load_from_db_optional(context, lastmsg_id)
                 .await
-                .context("loading message failed")?;
+                .context("Loading message failed")?
+        } else {
+            None
+        };
+
+        let lastcontact = if let Some(lastmsg) = &lastmsg {
             if lastmsg.from_id == ContactId::SELF {
-                (Some(lastmsg), None)
+                None
             } else {
                 match chat.typ {
                     Chattype::Group | Chattype::Broadcast | Chattype::Mailinglist => {
                         let lastcontact = Contact::get_by_id(context, lastmsg.from_id)
                             .await
                             .context("loading contact failed")?;
-                        (Some(lastmsg), Some(lastcontact))
+                        Some(lastcontact)
                     }
-                    Chattype::Single => (Some(lastmsg), None),
+                    Chattype::Single => None,
                 }
             }
         } else {
-            (None, None)
+            None
         };
 
         if chat.id.is_archived_link() {
@@ -476,7 +483,6 @@ mod tests {
         add_contact_to_chat, create_group_chat, get_chat_contacts, remove_contact_from_chat,
         send_text_msg, ProtectionStatus,
     };
-    use crate::message::Viewtype;
     use crate::receive_imf::receive_imf;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
@@ -510,8 +516,7 @@ mod tests {
         // Instead of setting drafts for chat_id1 and chat_id3, we could also sleep
         // 2s here.
         for chat_id in &[chat_id1, chat_id3, chat_id2] {
-            let mut msg = Message::new(Viewtype::Text);
+            let mut msg = Message::new_text("hello".to_string());
-            msg.set_text("hello".to_string());
             chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
         }
 
@@ -755,8 +760,7 @@ mod tests {
             .await
             .unwrap();
 
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("foo:\nbar \r\n test".to_string());
-        msg.set_text("foo:\nbar \r\n test".to_string());
         chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
@@ -764,6 +768,25 @@ mod tests {
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
 
+    /// Tests that summary does not fail to load
+    /// if the draft was deleted after loading the chatlist.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_summary_deleted_draft() {
+        let t = TestContext::new().await;
+
+        let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
+            .await
+            .unwrap();
+        let mut msg = Message::new_text("Foobar".to_string());
+        chat_id.set_draft(&t, Some(&mut msg)).await.unwrap();
+
+        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
+        chat_id.set_draft(&t, None).await.unwrap();
+
+        let summary_res = chats.get_summary(&t, 0, None).await;
+        assert!(summary_res.is_ok());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_load_broken() {
         let t = TestContext::new_bob().await;

src/config.rs

@@ -80,7 +80,7 @@ pub enum Config {
     /// SMTP server security (e.g. TLS, STARTTLS).
     SendSecurity,
 
-    /// Deprecated option for backwards compatibilty.
+    /// Deprecated option for backwards compatibility.
     ///
     /// Certificate checks for SMTP are actually controlled by `imap_certificate_checks` config.
     SmtpCertificateChecks,
@@ -143,7 +143,7 @@ pub enum Config {
     /// Send BCC copy to self.
     ///
     /// Should be enabled for multidevice setups.
-    #[strum(props(default = "1"))]
+    /// Default is 0 for chatmail accounts before a backup export, 1 otherwise.
     BccSelf,
 
     /// True if encryption is preferred according to Autocrypt standard.
@@ -202,7 +202,7 @@ pub enum Config {
     /// Value 1 is treated as "delete at once": messages are deleted
     /// immediately, without moving to DeltaChat folder.
     ///
-    /// Default is 1 for chatmail accounts before a backup export, 0 otherwise.
+    /// Default is 1 for chatmail accounts without `BccSelf`, 0 otherwise.
     DeleteServerAfter,
 
     /// Timer in seconds after which the message is deleted from the
@@ -396,6 +396,12 @@ pub enum Config {
     /// Make all outgoing messages with Autocrypt header "multipart/signed".
     SignUnencrypted,
 
+    /// Enable header protection for `Autocrypt` header.
+    ///
+    /// This is an experimental setting not compatible to other MUAs
+    /// and older Delta Chat versions (core version <= v1.149.0).
+    ProtectAutocrypt,
+
     /// Let the core save all events to the database.
     /// This value is used internally to remember the MsgId of the logging xdc
     #[strum(props(default = "0"))]
@@ -433,7 +439,14 @@ pub enum Config {
     WebxdcIntegration,
 
     /// Enable webxdc realtime features.
+    #[strum(props(default = "1"))]
     WebxdcRealtimeEnabled,
+
+    /// Last device token stored on the chatmail server.
+    ///
+    /// If it has not changed, we do not store
+    /// the device token again.
+    DeviceToken,
 }
 
 impl Config {
@@ -506,11 +519,19 @@ impl Context {
 
         // Default values
         let val = match key {
-            Config::ConfiguredInboxFolder => Some("INBOX"),
+            Config::BccSelf => match Box::pin(self.is_chatmail()).await? {
-            Config::DeleteServerAfter => match Box::pin(self.is_chatmail()).await? {
+                false => Some("1"),
-                false => Some("0"),
+                true => Some("0"),
-                true => Some("1"),
             },
+            Config::ConfiguredInboxFolder => Some("INBOX"),
+            Config::DeleteServerAfter => {
+                match !Box::pin(self.get_config_bool(Config::BccSelf)).await?
+                    && Box::pin(self.is_chatmail()).await?
+                {
+                    true => Some("1"),
+                    false => Some("0"),
+                }
+            }
             _ => key.get_str("default"),
         };
         Ok(val.map(|s| s.to_string()))
@@ -784,6 +805,12 @@ impl Context {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
+        if matches!(
+            key,
+            Config::Displayname | Config::Selfavatar | Config::PrivateTag
+        ) {
+            self.emit_event(EventType::AccountsItemChanged);
+        }
         if key.is_synced() {
             self.emit_event(EventType::ConfigSynced { key });
         }
@@ -1086,6 +1113,28 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_delete_server_after_default() -> Result<()> {
+        let t = &TestContext::new_alice().await;
+
+        // Check that the settings are displayed correctly.
+        assert_eq!(t.get_config(Config::BccSelf).await?, Some("1".to_string()));
+        assert_eq!(
+            t.get_config(Config::DeleteServerAfter).await?,
+            Some("0".to_string())
+        );
+
+        // Leaving emails on the server even w/o `BccSelf` is a good default at least because other
+        // MUAs do so even if the server doesn't save sent messages to some sentbox (like Gmail
+        // does).
+        t.set_config_bool(Config::BccSelf, false).await?;
+        assert_eq!(
+            t.get_config(Config::DeleteServerAfter).await?,
+            Some("0".to_string())
+        );
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;

src/configure.rs

@@ -19,11 +19,12 @@ use auto_outlook::outlk_autodiscover;
 use deltachat_contact_tools::EmailAddress;
 use futures::FutureExt;
 use futures_lite::FutureExt as _;
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
+use percent_encoding::utf8_percent_encode;
 use server_params::{expand_param_vector, ServerParams};
 use tokio::task;
 
 use crate::config::{self, Config};
+use crate::constants::NON_ALPHANUMERIC_WITHOUT_DOT;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::log::LogExt;
@@ -31,14 +32,14 @@ use crate::login_param::{
     ConfiguredCertificateChecks, ConfiguredLoginParam, ConfiguredServerLoginParam,
     ConnectionCandidate, EnteredCertificateChecks, EnteredLoginParam,
 };
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
-use crate::stock_str;
 use crate::sync::Sync::*;
 use crate::tools::time;
 use crate::{chat, e2ee, provider};
+use crate::{stock_str, EventType};
 use deltachat_contact_tools::addr_cmp;
 
 macro_rules! progress {
@@ -111,15 +112,10 @@ impl Context {
 
         let param = EnteredLoginParam::load(self).await?;
         let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-
+        let configured_param = configure(self, &param).await?;
-        let configured_param_res = configure(self, &param).await;
-        self.set_config_internal(Config::NotifyAboutWrongPw, None)
-            .await?;
-
-        on_configure_completed(self, configured_param_res?, old_addr).await?;
-
         self.set_config_internal(Config::NotifyAboutWrongPw, Some("1"))
             .await?;
+        on_configure_completed(self, configured_param, old_addr).await?;
         Ok(())
     }
 }
@@ -147,8 +143,7 @@ async fn on_configure_completed(
         }
 
         if !provider.after_login_hint.is_empty() {
-            let mut msg = Message::new(Viewtype::Text);
+            let mut msg = Message::new_text(provider.after_login_hint.to_string());
-            msg.text = provider.after_login_hint.to_string();
             if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg))
                 .await
                 .is_err()
@@ -161,9 +156,9 @@ async fn on_configure_completed(
     if let Some(new_addr) = context.get_config(Config::ConfiguredAddr).await? {
         if let Some(old_addr) = old_addr {
             if !addr_cmp(&new_addr, &old_addr) {
-                let mut msg = Message::new(Viewtype::Text);
+                let mut msg = Message::new_text(
-                msg.text =
+                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await,
-                    stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await;
+                );
                 chat::add_device_msg(context, None, Some(&mut msg))
                     .await
                     .context("Cannot add AEAP explanation")
@@ -417,7 +412,8 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
         configured_param.oauth2,
         r,
     );
-    let mut imap_session = match imap.connect(ctx).await {
+    let configuring = true;
+    let mut imap_session = match imap.connect(ctx, configuring).await {
         Ok(session) => session,
         Err(err) => bail!("{}", nicer_configuration_error(ctx, err.to_string()).await),
     };
@@ -456,8 +452,9 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
     imap.configure_folders(ctx, &mut imap_session, create_mvbox)
         .await?;
 
+    let create = true;
     imap_session
-        .select_with_uidvalidity(ctx, "INBOX")
+        .select_with_uidvalidity(ctx, "INBOX", create)
         .await
         .context("could not read INBOX status")?;
 
@@ -490,6 +487,7 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Configure
     update_device_chats_handle.await??;
 
     ctx.sql.set_raw_config_bool("configured", true).await?;
+    ctx.emit_event(EventType::AccountsItemChanged);
 
     Ok(configured_param)
 }
@@ -503,7 +501,15 @@ async fn get_autoconfig(
     param: &EnteredLoginParam,
     param_domain: &str,
 ) -> Option<Vec<ServerParams>> {
-    let param_addr_urlencoded = utf8_percent_encode(&param.addr, NON_ALPHANUMERIC).to_string();
+    // Make sure to not encode `.` as `%2E` here.
+    // Some servers like murena.io on 2024-11-01 produce incorrect autoconfig XML
+    // when address is encoded.
+    // E.g.
+    // <https://autoconfig.murena.io/mail/config-v1.1.xml?emailaddress=foobar%40example%2Eorg>
+    // produced XML file with `<username>foobar@example%2Eorg</username>`
+    // resulting in failure to log in.
+    let param_addr_urlencoded =
+        utf8_percent_encode(&param.addr, NON_ALPHANUMERIC_WITHOUT_DOT).to_string();
 
     if let Ok(res) = moz_autoconfigure(
         ctx,
@@ -607,8 +613,6 @@ pub enum Error {
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
     use crate::config::Config;
     use crate::login_param::EnteredServerLoginParam;

src/configure/auto_mozilla.rs

@@ -67,19 +67,15 @@ fn parse_server<B: BufRead>(
 
     let typ = server_event
         .attributes()
-        .find(|attr| {
+        .find_map(|attr| {
-            attr.as_ref()
+            attr.ok().filter(|a| {
-                .map(|a| {
                 String::from_utf8_lossy(a.key.as_ref())
                     .trim()
-                        .to_lowercase()
+                    .eq_ignore_ascii_case("type")
-                        == "type"
             })
-                .unwrap_or_default()
         })
         .map(|typ| {
-            typ.unwrap()
+            typ.decode_and_unescape_value(reader.decoder())
-                .decode_and_unescape_value(reader.decoder())
                 .unwrap_or_default()
                 .to_lowercase()
         })
@@ -272,8 +268,6 @@ pub(crate) async fn moz_autoconfigure(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/configure/auto_outlook.rs

@@ -215,8 +215,6 @@ pub(crate) async fn outlk_autodiscover(
 
 #[cfg(test)]
 mod tests {
-    #![allow(clippy::indexing_slicing)]
-
     use super::*;
 
     #[test]

src/constants.rs

@@ -4,12 +4,16 @@
 
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
+use percent_encoding::{AsciiSet, NON_ALPHANUMERIC};
 use serde::{Deserialize, Serialize};
 
 use crate::chat::ChatId;
 
 pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
+/// Set of characters to percent-encode in email addresses and names.
+pub(crate) const NON_ALPHANUMERIC_WITHOUT_DOT: &AsciiSet = &NON_ALPHANUMERIC.remove(b'.');
+
 #[derive(
     Debug,
     Default,

src/contact.rs

@@ -129,20 +129,54 @@ impl ContactId {
     ) -> Result<()> {
         context
             .sql
-            .execute(
+            .transaction(|transaction| {
-                &format!(
+                let mut stmt = transaction
-                    "UPDATE contacts SET origin=? WHERE id IN ({}) AND origin<?",
+                    .prepare("UPDATE contacts SET origin=?1 WHERE id = ?2 AND origin < ?1")?;
-                    sql::repeat_vars(ids.len())
+                for id in ids {
-                ),
+                    stmt.execute((origin, id))?;
-                rusqlite::params_from_iter(
+                }
-                    params_iter(&[origin])
+                Ok(())
-                        .chain(params_iter(ids))
+            })
-                        .chain(params_iter(&[origin])),
-                ),
-            )
             .await?;
         Ok(())
     }
+
+    /// Returns contact address.
+    pub async fn addr(&self, context: &Context) -> Result<String> {
+        let addr = context
+            .sql
+            .query_row("SELECT addr FROM contacts WHERE id=?", (self,), |row| {
+                let addr: String = row.get(0)?;
+                Ok(addr)
+            })
+            .await?;
+        Ok(addr)
+    }
+
+    /// Resets encryption with the contact.
+    ///
+    /// Effect is similar to receiving a message without Autocrypt header
+    /// from the contact, but this action is triggered manually by the user.
+    ///
+    /// For example, this will result in sending the next message
+    /// to 1:1 chat unencrypted, but will not remove existing verified keys.
+    pub async fn reset_encryption(self, context: &Context) -> Result<()> {
+        let now = time();
+
+        let addr = self.addr(context).await?;
+        if let Some(mut peerstate) = Peerstate::from_addr(context, &addr).await? {
+            peerstate.degrade_encryption(now);
+            peerstate.save_to_db(&context.sql).await?;
+        }
+
+        // Reset 1:1 chat protection.
+        if let Some(chat_id) = ChatId::lookup_by_contact(context, self).await? {
+            chat_id
+                .set_protection(context, ProtectionStatus::Unprotected, now, Some(self))
+                .await?;
+        }
+        Ok(())
+    }
 }
 
 impl fmt::Display for ContactId {
@@ -425,9 +459,12 @@ pub enum Origin {
     /// To: of incoming messages of unknown sender
     IncomingUnknownTo = 0x40,
 
-    /// address scanned but not verified
+    /// Address scanned but not verified.
     UnhandledQrScan = 0x80,
 
+    /// Address scanned from a SecureJoin QR code, but not verified yet.
+    UnhandledSecurejoinQrScan = 0x81,
+
     /// Reply-To: of incoming message of known sender
     /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
@@ -1212,15 +1249,15 @@ impl Contact {
 
         let fingerprint_self = load_self_public_key(context)
             .await?
-            .fingerprint()
+            .dc_fingerprint()
             .to_string();
         let fingerprint_other_verified = peerstate
             .peek_key(true)
-            .map(|k| k.fingerprint().to_string())
+            .map(|k| k.dc_fingerprint().to_string())
             .unwrap_or_default();
         let fingerprint_other_unverified = peerstate
             .peek_key(false)
-            .map(|k| k.fingerprint().to_string())
+            .map(|k| k.dc_fingerprint().to_string())
             .unwrap_or_default();
         if addr < peerstate.addr {
             cat_fingerprint(&mut ret, &addr, &fingerprint_self, "");
@@ -1944,7 +1981,7 @@ mod tests {
     use crate::chat::{get_chat_contacts, send_text_msg, Chat};
     use crate::chatlist::Chatlist;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::{self, TestContext, TestContextManager};
+    use crate::test_utils::{self, TestContext, TestContextManager, TimeShiftFalsePositiveNote};
 
     #[test]
     fn test_contact_id_values() {
@@ -2875,6 +2912,8 @@ Hi."#;
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_was_seen_recently() -> Result<()> {
+        let _n = TimeShiftFalsePositiveNote;
+
         let mut tcm = TestContextManager::new();
         let alice = tcm.alice().await;
         let bob = tcm.bob().await;
@@ -2890,18 +2929,7 @@ Hi."#;
         bob.recv_msg(&sent_msg).await;
         let contact = Contact::get_by_id(&bob, *contacts.first().unwrap()).await?;
 
-        let green = nu_ansi_term::Color::Green.normal();
+        assert!(contact.was_seen_recently());
-        assert!(
-            contact.was_seen_recently(),
-            "{}",
-            green.paint(
-                "\nNOTE: This test failure is probably a false-positive, caused by tests running in parallel.
-The issue is that `SystemTime::shift()` (a utility function for tests) changes the time for all threads doing tests, and not only for the running test.
-Until the false-positive is fixed:
-- Use `cargo test -- --test-threads 1` instead of `cargo test`
-- Or use `cargo nextest run` (install with `cargo install cargo-nextest --locked`)\n"
-            )
-        );
 
         let self_contact = Contact::get_by_id(&bob, ContactId::SELF).await?;
         assert!(!self_contact.was_seen_recently());
@@ -3149,4 +3177,75 @@ Until the false-positive is fixed:
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_reset_encryption() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        let msg = tcm.send_recv_accept(alice, bob, "Hello!").await;
+        assert_eq!(msg.get_showpadlock(), false);
+
+        let msg = tcm.send_recv(bob, alice, "Hi!").await;
+        assert_eq!(msg.get_showpadlock(), true);
+        let alice_bob_contact_id = msg.from_id;
+
+        alice_bob_contact_id.reset_encryption(alice).await?;
+
+        let msg = tcm.send_recv(alice, bob, "Unencrypted").await;
+        assert_eq!(msg.get_showpadlock(), false);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_reset_verified_encryption() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        tcm.execute_securejoin(bob, alice).await;
+
+        let msg = tcm.send_recv(bob, alice, "Encrypted").await;
+        assert_eq!(msg.get_showpadlock(), true);
+
+        let alice_bob_chat_id = msg.chat_id;
+        let alice_bob_contact_id = msg.from_id;
+        alice_bob_contact_id.reset_encryption(alice).await?;
+
+        // Check that the contact is still verified after resetting encryption.
+        let alice_bob_contact = Contact::get_by_id(alice, alice_bob_contact_id).await?;
+        assert_eq!(alice_bob_contact.is_verified(alice).await?, true);
+
+        // 1:1 chat and profile is no longer verified.
+        assert_eq!(alice_bob_contact.is_profile_verified(alice).await?, false);
+
+        let info_msg = alice.get_last_msg_in(alice_bob_chat_id).await;
+        assert_eq!(
+            info_msg.text,
+            "bob@example.net sent a message from another device."
+        );
+
+        let msg = tcm.send_recv(alice, bob, "Unencrypted").await;
+        assert_eq!(msg.get_showpadlock(), false);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_self_is_verified() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+
+        let contact = Contact::get_by_id(&alice, ContactId::SELF).await?;
+        assert_eq!(contact.is_verified(&alice).await?, true);
+        assert!(contact.is_profile_verified(&alice).await?);
+        assert!(contact.get_verifier_id(&alice).await?.is_none());
+
+        let chat_id = ChatId::get_for_contact(&alice, ContactId::SELF).await?;
+        assert!(chat_id.is_protected(&alice).await.unwrap() == ProtectionStatus::Protected);
+
+        Ok(())
+    }
 }

src/context.rs

@@ -10,9 +10,10 @@ use std::time::Duration;
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
+use pgp::types::PublicKeyTrait;
 use pgp::SignedPublicKey;
 use ratelimit::Ratelimit;
-use tokio::sync::{Mutex, Notify, OnceCell, RwLock};
+use tokio::sync::{Mutex, Notify, RwLock};
 
 use crate::aheader::EncryptPreference;
 use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus};
@@ -28,7 +29,7 @@ use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::imap::{FolderMeaning, Imap, ServerMetadata};
 use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _};
 use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam};
-use crate::message::{self, Message, MessageState, MsgId, Viewtype};
+use crate::message::{self, Message, MessageState, MsgId};
 use crate::param::{Param, Params};
 use crate::peer_channels::Iroh;
 use crate::peerstate::Peerstate;
@@ -275,7 +276,7 @@ pub struct InnerContext {
     /// The text of the last error logged and emitted as an event.
     /// If the ui wants to display an error after a failure,
     /// `last_error` should be used to avoid races with the event thread.
-    pub(crate) last_error: std::sync::RwLock<String>,
+    pub(crate) last_error: parking_lot::RwLock<String>,
 
     /// If debug logging is enabled, this contains all necessary information
     ///
@@ -291,7 +292,7 @@ pub struct InnerContext {
     pub(crate) push_subscribed: AtomicBool,
 
     /// Iroh for realtime peer channels.
-    pub(crate) iroh: OnceCell<Iroh>,
+    pub(crate) iroh: Arc<RwLock<Option<Iroh>>>,
 }
 
 /// The state of ongoing process.
@@ -445,11 +446,11 @@ impl Context {
             metadata: RwLock::new(None),
             creation_time: tools::Time::now(),
             last_full_folder_scan: Mutex::new(None),
-            last_error: std::sync::RwLock::new("".to_string()),
+            last_error: parking_lot::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
             push_subscriber,
             push_subscribed: AtomicBool::new(false),
-            iroh: OnceCell::new(),
+            iroh: Arc::new(RwLock::new(None)),
         };
 
         let ctx = Context {
@@ -471,12 +472,33 @@ impl Context {
             // Allow at least 1 message every second + a burst of 3.
             *lock = Ratelimit::new(Duration::new(3, 0), 3.0);
         }
+
+        // The next line is mainly for iOS:
+        // iOS starts a separate process for receiving notifications and if the user concurrently
+        // starts the app, the UI process opens the database but waits with calling start_io()
+        // until the notifications process finishes.
+        // Now, some configs may have changed, so, we need to invalidate the cache.
+        self.sql.config_cache.write().await.clear();
+
         self.scheduler.start(self.clone()).await;
     }
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         self.scheduler.stop(self).await;
+        if let Some(iroh) = self.iroh.write().await.take() {
+            // Close all QUIC connections.
+
+            // Spawn into a separate task,
+            // because Iroh calls `wait_idle()` internally
+            // and it may take time, especially if the network
+            // has become unavailable.
+            tokio::spawn(async move {
+                // We do not log the error because we do not want the task
+                // to hold the reference to Context.
+                let _ = tokio::time::timeout(Duration::from_secs(60), iroh.close()).await;
+            });
+        }
     }
 
     /// Restarts the IO scheduler if it was running before
@@ -487,7 +509,7 @@ impl Context {
 
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        if let Some(iroh) = self.iroh.get() {
+        if let Some(ref iroh) = *self.iroh.read().await {
             iroh.network_change().await;
         }
         self.scheduler.maybe_network().await;
@@ -531,23 +553,7 @@ impl Context {
 
         if self.scheduler.is_running().await {
             self.scheduler.maybe_network().await;
-
+            self.wait_for_all_work_done().await;
-            // Wait until fetching is finished.
-            // Ideally we could wait for connectivity change events,
-            // but sleep loop is good enough.
-
-            // First 100 ms sleep in chunks of 10 ms.
-            for _ in 0..10 {
-                if self.all_work_done().await {
-                    break;
-                }
-                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
-            }
-
-            // If we are not finished in 100 ms, keep waking up every 100 ms.
-            while !self.all_work_done().await {
-                tokio::time::sleep(std::time::Duration::from_millis(100)).await;
-            }
         } else {
             // Pause the scheduler to ensure another connection does not start
             // while we are fetching on a dedicated connection.
@@ -637,14 +643,36 @@ impl Context {
     }
 
     /// Emits a MsgsChanged event with specified chat and message ids
+    ///
+    /// If IDs are unset, [`Self::emit_msgs_changed_without_ids`]
+    /// or [`Self::emit_msgs_changed_without_msg_id`] should be used
+    /// instead of this function.
     pub fn emit_msgs_changed(&self, chat_id: ChatId, msg_id: MsgId) {
+        debug_assert!(!chat_id.is_unset());
+        debug_assert!(!msg_id.is_unset());
+
         self.emit_event(EventType::MsgsChanged { chat_id, msg_id });
         chatlist_events::emit_chatlist_changed(self);
         chatlist_events::emit_chatlist_item_changed(self, chat_id);
     }
 
+    /// Emits a MsgsChanged event with specified chat and without message id.
+    pub fn emit_msgs_changed_without_msg_id(&self, chat_id: ChatId) {
+        debug_assert!(!chat_id.is_unset());
+
+        self.emit_event(EventType::MsgsChanged {
+            chat_id,
+            msg_id: MsgId::new(0),
+        });
+        chatlist_events::emit_chatlist_changed(self);
+        chatlist_events::emit_chatlist_item_changed(self, chat_id);
+    }
+
     /// Emits an IncomingMsg event with specified chat and message ids
     pub fn emit_incoming_msg(&self, chat_id: ChatId, msg_id: MsgId) {
+        debug_assert!(!chat_id.is_unset());
+        debug_assert!(!msg_id.is_unset());
+
         self.emit_event(EventType::IncomingMsg { chat_id, msg_id });
         chatlist_events::emit_chatlist_changed(self);
         chatlist_events::emit_chatlist_item_changed(self, chat_id);
@@ -781,7 +809,7 @@ impl Context {
             .count("SELECT COUNT(*) FROM acpeerstates;", ())
             .await?;
         let fingerprint_str = match load_self_public_key(self).await {
-            Ok(key) => key.fingerprint().hex(),
+            Ok(key) => key.dc_fingerprint().hex(),
             Err(err) => format!("<key failure: {err}>"),
         };
 
@@ -990,6 +1018,12 @@ impl Context {
                 .await?
                 .to_string(),
         );
+        res.insert(
+            "protect_autocrypt",
+            self.get_config_int(Config::ProtectAutocrypt)
+                .await?
+                .to_string(),
+        );
         res.insert(
             "debug_logging",
             self.get_config_int(Config::DebugLogging).await?.to_string(),
@@ -1171,15 +1205,14 @@ impl Context {
             EncryptPreference::Mutual,
             &public_key,
         );
-        let fingerprint = public_key.fingerprint();
+        let fingerprint = public_key.dc_fingerprint();
         peerstate.set_verified(public_key, fingerprint, "".to_string())?;
         peerstate.save_to_db(&self.sql).await?;
         chat_id
             .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id))
             .await?;
 
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text(self.get_self_report().await?);
-        msg.text = self.get_self_report().await?;
 
         chat_id.set_draft(self, Some(&mut msg)).await?;
 
@@ -1744,6 +1777,7 @@ mod tests {
             "socks5_password",
             "key_id",
             "webxdc_integration",
+            "device_token",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -1778,12 +1812,10 @@ mod tests {
         assert!(res.is_empty());
 
         // Add messages to chat with Bob.
-        let mut msg1 = Message::new(Viewtype::Text);
+        let mut msg1 = Message::new_text("foobar".to_string());
-        msg1.set_text("foobar".to_string());
         send_msg(&alice, chat.id, &mut msg1).await?;
 
-        let mut msg2 = Message::new(Viewtype::Text);
+        let mut msg2 = Message::new_text("barbaz".to_string());
-        msg2.set_text("barbaz".to_string());
         send_msg(&alice, chat.id, &mut msg2).await?;
 
         alice.send_text(chat.id, "Δ-Chat").await;
@@ -1886,8 +1918,7 @@ mod tests {
             .await;
 
         // Add 999 messages
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("foobar".to_string());
-        msg.set_text("foobar".to_string());
         for _ in 0..999 {
             send_msg(&alice, chat.id, &mut msg).await?;
         }
@@ -2061,4 +2092,41 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_cache_is_cleared_when_io_is_started() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        assert_eq!(
+            alice.get_config(Config::ShowEmails).await?,
+            Some("2".to_string())
+        );
+
+        // Change the config circumventing the cache
+        // This simulates what the notification plugin on iOS might do
+        // because it runs in a different process
+        alice
+            .sql
+            .execute(
+                "INSERT OR REPLACE INTO config (keyname, value) VALUES ('show_emails', '0')",
+                (),
+            )
+            .await?;
+
+        // Alice's Delta Chat doesn't know about it yet:
+        assert_eq!(
+            alice.get_config(Config::ShowEmails).await?,
+            Some("2".to_string())
+        );
+
+        // Starting IO will fail of course because no server settings are configured,
+        // but it should invalidate the caches:
+        alice.start_io().await;
+
+        assert_eq!(
+            alice.get_config(Config::ShowEmails).await?,
+            Some("0".to_string())
+        );
+
+        Ok(())
+    }
 }

src/debug_logging.rs

@@ -60,9 +60,11 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                         "time": time,
                     }),
                     info: None,
+                    href: None,
                     summary: None,
                     document: None,
                     uid: None,
+                    notify: None,
                 },
                 time,
             )

src/decrypt.rs

@@ -1,125 +1,36 @@
 //! End-to-end decryption support.
 
 use std::collections::HashSet;
-use std::str::FromStr;
 
 use anyhow::Result;
 use deltachat_contact_tools::addr_cmp;
 use mailparse::ParsedMail;
 
 use crate::aheader::Aheader;
-use crate::authres::handle_authres;
-use crate::authres::{self, DkimResults};
 use crate::context::Context;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::peerstate::Peerstate;
 use crate::pgp;
 
 /// Tries to decrypt a message, but only if it is structured as an Autocrypt message.
 ///
-/// If successful and the message is encrypted, returns decrypted body and a set of valid
+/// If successful and the message is encrypted, returns decrypted body.
-/// signature fingerprints.
-///
-/// If the message is wrongly signed, HashSet will be empty.
 pub fn try_decrypt(
     mail: &ParsedMail<'_>,
     private_keyring: &[SignedSecretKey],
-    public_keyring_for_validate: &[SignedPublicKey],
+) -> Result<Option<::pgp::composed::Message>> {
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
     let Some(encrypted_data_part) = get_encrypted_mime(mail) else {
         return Ok(None);
     };
 
-    decrypt_part(
+    let data = encrypted_data_part.get_body_raw()?;
-        encrypted_data_part,
+    let msg = pgp::pk_decrypt(data, private_keyring)?;
-        private_keyring,
-        public_keyring_for_validate,
-    )
-}
 
-pub(crate) async fn prepare_decryption(
+    Ok(Some(msg))
-    context: &Context,
-    mail: &ParsedMail<'_>,
-    from: &str,
-    message_time: i64,
-) -> Result<DecryptionInfo> {
-    if mail.headers.get_header(HeaderDef::ListPost).is_some() {
-        if mail.headers.get_header(HeaderDef::Autocrypt).is_some() {
-            info!(
-                context,
-                "Ignoring autocrypt header since this is a mailing list message. \
-                NOTE: For privacy reasons, the mailing list software should remove Autocrypt headers."
-            );
-        }
-        return Ok(DecryptionInfo {
-            from: from.to_string(),
-            autocrypt_header: None,
-            peerstate: None,
-            message_time,
-            dkim_results: DkimResults { dkim_passed: false },
-        });
-    }
-
-    let autocrypt_header = if context.is_self_addr(from).await? {
-        None
-    } else if let Some(aheader_value) = mail.headers.get_header_value(HeaderDef::Autocrypt) {
-        match Aheader::from_str(&aheader_value) {
-            Ok(header) if addr_cmp(&header.addr, from) => Some(header),
-            Ok(header) => {
-                warn!(
-                    context,
-                    "Autocrypt header address {:?} is not {:?}.", header.addr, from
-                );
-                None
-            }
-            Err(err) => {
-                warn!(context, "Failed to parse Autocrypt header: {:#}.", err);
-                None
-            }
-        }
-    } else {
-        None
-    };
-
-    let dkim_results = handle_authres(context, mail, from).await?;
-    let allow_aeap = get_encrypted_mime(mail).is_some();
-    let peerstate = get_autocrypt_peerstate(
-        context,
-        from,
-        autocrypt_header.as_ref(),
-        message_time,
-        allow_aeap,
-    )
-    .await?;
-
-    Ok(DecryptionInfo {
-        from: from.to_string(),
-        autocrypt_header,
-        peerstate,
-        message_time,
-        dkim_results,
-    })
-}
-
-#[derive(Debug)]
-pub struct DecryptionInfo {
-    /// The From address. This is the address from the unnencrypted, outer
-    /// From header.
-    pub from: String,
-    pub autocrypt_header: Option<Aheader>,
-    /// The peerstate that will be used to validate the signatures
-    pub peerstate: Option<Peerstate>,
-    /// The timestamp when the message was sent.
-    /// If this is older than the peerstate's last_seen, this probably
-    /// means out-of-order message arrival, We don't modify the
-    /// peerstate in this case.
-    pub message_time: i64,
-    pub(crate) dkim_results: authres::DkimResults,
 }
 
 /// Returns a reference to the encrypted payload of a message.
-fn get_encrypted_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+pub(crate) fn get_encrypted_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
     get_autocrypt_mime(mail)
         .or_else(|| get_mixed_up_mime(mail))
         .or_else(|| get_attachment_mime(mail))
@@ -204,37 +115,6 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail
     }
 }
 
-/// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    mail: &ParsedMail<'_>,
-    private_keyring: &[SignedSecretKey],
-    public_keyring_for_validate: &[SignedPublicKey],
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let data = mail.get_body_raw()?;
-
-    if has_decrypted_pgp_armor(&data) {
-        let (plain, ret_valid_signatures) =
-            pgp::pk_decrypt(data, private_keyring, public_keyring_for_validate)?;
-        return Ok(Some((plain, ret_valid_signatures)));
-    }
-
-    Ok(None)
-}
-
-#[allow(clippy::indexing_slicing)]
-fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
-    if let Some(index) = input.iter().position(|b| *b > b' ') {
-        if input.len() - index > 26 {
-            let start = index;
-            let end = start + 27;
-
-            return &input[start..end] == b"-----BEGIN PGP MESSAGE-----";
-        }
-    }
-
-    false
-}
-
 /// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
 ///
 /// Returns the signed part and the set of key
@@ -302,7 +182,7 @@ pub(crate) async fn get_autocrypt_peerstate(
             // if the fingerprint is verified.
             peerstate = Peerstate::from_verified_fingerprint_or_addr(
                 context,
-                &header.public_key.fingerprint(),
+                &header.public_key.dc_fingerprint(),
                 from,
             )
             .await?;
@@ -346,24 +226,6 @@ mod tests {
     use crate::receive_imf::receive_imf;
     use crate::test_utils::TestContext;
 
-    #[test]
-    fn test_has_decrypted_pgp_armor() {
-        let data = b" -----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"    \n-----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"    -----BEGIN PGP MESSAGE---";
-        assert_eq!(has_decrypted_pgp_armor(data), false);
-
-        let data = b" -----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"blas";
-        assert_eq!(has_decrypted_pgp_armor(data), false);
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_mixed_up_mime() -> Result<()> {
         // "Mixed Up" mail as received when sending an encrypted

src/download.rs

@@ -3,7 +3,7 @@
 use std::cmp::max;
 use std::collections::BTreeMap;
 
-use anyhow::{anyhow, bail, Result};
+use anyhow::{anyhow, bail, ensure, Result};
 use deltachat_derive::{FromSql, ToSql};
 use serde::{Deserialize, Serialize};
 
@@ -201,7 +201,11 @@ impl Session {
             bail!("Attempt to fetch UID 0");
         }
 
-        self.select_with_uidvalidity(context, folder).await?;
+        let create = false;
+        let folder_exists = self
+            .select_with_uidvalidity(context, folder, create)
+            .await?;
+        ensure!(folder_exists, "No folder {folder}");
 
         // we are connected, and the folder is selected
         info!(context, "Downloading message {}/{} fully...", folder, uid);
@@ -318,8 +322,7 @@ mod tests {
         let t = TestContext::new_alice().await;
         let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
 
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("Hi Bob".to_owned());
-        msg.set_text("Hi Bob".to_owned());
         let msg_id = send_msg(&t, chat.id, &mut msg).await?;
         let msg = Message::load_from_db(&t, msg_id).await?;
         assert_eq!(msg.download_state(), DownloadState::Done);
@@ -441,7 +444,7 @@ mod tests {
         let _sent1 = alice.send_msg(chat_id, &mut instance).await;
 
         alice
-            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#, "d")
+            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#)
             .await?;
         alice.flush_status_updates().await?;
         let sent2 = alice.pop_sent_msg().await;

src/e2ee.rs

@@ -303,12 +303,12 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             last_seen_autocrypt: 14,
             prefer_encrypt,
             public_key: Some(pub_key.clone()),
-            public_key_fingerprint: Some(pub_key.fingerprint()),
+            public_key_fingerprint: Some(pub_key.dc_fingerprint()),
             gossip_key: Some(pub_key.clone()),
             gossip_timestamp: 15,
-            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key_fingerprint: Some(pub_key.dc_fingerprint()),
             verified_key: Some(pub_key.clone()),
-            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key_fingerprint: Some(pub_key.dc_fingerprint()),
             verifier: None,
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,

src/ephemeral.rs

@@ -84,7 +84,6 @@ use crate::location;
 use crate::log::LogExt;
 use crate::message::{Message, MessageState, MsgId, Viewtype};
 use crate::mimeparser::SystemMessage;
-use crate::sql::{self, params_iter};
 use crate::stock_str;
 use crate::tools::{duration_to_str, time, SystemTime};
 
@@ -223,8 +222,9 @@ impl ChatId {
         self.inner_set_ephemeral_timer(context, timer).await?;
 
         if self.is_promoted(context).await? {
-            let mut msg = Message::new(Viewtype::Text);
+            let mut msg = Message::new_text(
-            msg.text = stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await;
+                stock_ephemeral_timer_changed(context, timer, ContactId::SELF).await,
+            );
             msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
             if let Err(err) = send_msg(context, self, &mut msg).await {
                 error!(
@@ -328,23 +328,44 @@ pub(crate) async fn start_ephemeral_timers_msgids(
     msg_ids: &[MsgId],
 ) -> Result<()> {
     let now = time();
-    let count = context
+    let should_interrupt =
+    context
+        .sql
+        .transaction(move |transaction| {
+            let mut should_interrupt = false;
+            let mut stmt =
+                transaction.prepare(
+                    "UPDATE msgs SET ephemeral_timestamp = ?1 + ephemeral_timer
+                     WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?1 + ephemeral_timer) AND ephemeral_timer > 0
+                     AND id=?2")?;
+            for msg_id in msg_ids {
+                should_interrupt |= stmt.execute((now, msg_id))? > 0;
+            }
+            Ok(should_interrupt)
+        }).await?;
+    if should_interrupt {
+        context.scheduler.interrupt_ephemeral_task().await;
+    }
+    Ok(())
+}
+
+/// Starts ephemeral timer for all messages in the chat.
+///
+/// This should be called when chat is marked as noticed.
+pub(crate) async fn start_chat_ephemeral_timers(context: &Context, chat_id: ChatId) -> Result<()> {
+    let now = time();
+    let should_interrupt = context
         .sql
         .execute(
-            &format!(
+            "UPDATE msgs SET ephemeral_timestamp = ?1 + ephemeral_timer
-                "UPDATE msgs SET ephemeral_timestamp = ? + ephemeral_timer
+             WHERE chat_id = ?2
-         WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ? + ephemeral_timer) AND ephemeral_timer > 0
+             AND ephemeral_timer > 0
-         AND id IN ({})",
+             AND (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?1 + ephemeral_timer)",
-                sql::repeat_vars(msg_ids.len())
+            (now, chat_id),
-            ),
-            rusqlite::params_from_iter(
-                std::iter::once(&now as &dyn crate::sql::ToSql)
-                    .chain(std::iter::once(&now as &dyn crate::sql::ToSql))
-                    .chain(params_iter(msg_ids)),
-            ),
         )
-        .await?;
+        .await?
-    if count > 0 {
+        > 0;
+    if should_interrupt {
         context.scheduler.interrupt_ephemeral_task().await;
     }
     Ok(())
@@ -481,7 +502,7 @@ pub(crate) async fn delete_expired_messages(context: &Context, now: i64) -> Resu
         }
 
         for modified_chat_id in modified_chat_ids {
-            context.emit_msgs_changed(modified_chat_id, MsgId::new(0));
+            context.emit_msgs_changed_without_msg_id(modified_chat_id);
         }
 
         for msg_id in webxdc_deleted {
@@ -694,7 +715,9 @@ pub(crate) async fn start_ephemeral_timers(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chat::{marknoticed_chat, set_muted, ChatVisibility, MuteDuration};
     use crate::config::Config;
+    use crate::constants::DC_CHAT_ID_ARCHIVED_LINK;
     use crate::download::DownloadState;
     use crate::location;
     use crate::message::markseen_msgs;
@@ -929,7 +952,6 @@ mod tests {
 
         // Alice sends a text message.
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
 
@@ -956,14 +978,12 @@ mod tests {
 
         // Alice sends message to Bob
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
         bob.recv_msg(&sent).await;
 
         // Alice sends second message to Bob, with no timer
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
         chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
         let sent = alice.pop_sent_msg().await;
 
@@ -1362,8 +1382,7 @@ mod tests {
         chat.id
             .set_ephemeral_timer(&alice, Timer::Enabled { duration })
             .await?;
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("hi".to_string());
-        msg.set_text("hi".to_string());
         assert!(chat::send_msg_sync(&alice, chat.id, &mut msg)
             .await
             .is_err());
@@ -1393,8 +1412,7 @@ mod tests {
         let sent = alice.pop_sent_msg().await;
         bob.recv_msg(&sent).await;
 
-        let mut poi_msg = Message::new(Viewtype::Text);
+        let mut poi_msg = Message::new_text("Here".to_string());
-        poi_msg.text = "Here".to_string();
         poi_msg.set_location(10.0, 20.0);
 
         let alice_sent_message = alice.send_msg(chat.id, &mut poi_msg).await;
@@ -1426,4 +1444,77 @@ mod tests {
 
         Ok(())
     }
+
+    /// Tests that ephemeral timer is started when the chat is noticed.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_noticed_ephemeral_timer() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        let chat = alice.create_chat(bob).await;
+        let duration = 60;
+        chat.id
+            .set_ephemeral_timer(alice, Timer::Enabled { duration })
+            .await?;
+        let bob_received_message = tcm.send_recv(alice, bob, "Hello!").await;
+
+        marknoticed_chat(bob, bob_received_message.chat_id).await?;
+        SystemTime::shift(Duration::from_secs(100));
+
+        delete_expired_messages(bob, time()).await?;
+
+        assert!(Message::load_from_db_optional(bob, bob_received_message.id)
+            .await?
+            .is_none());
+        Ok(())
+    }
+
+    /// Tests that archiving the chat starts ephemeral timer.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_archived_ephemeral_timer() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = &tcm.alice().await;
+        let bob = &tcm.bob().await;
+
+        let chat = alice.create_chat(bob).await;
+        let duration = 60;
+        chat.id
+            .set_ephemeral_timer(alice, Timer::Enabled { duration })
+            .await?;
+        let bob_received_message = tcm.send_recv(alice, bob, "Hello!").await;
+
+        bob_received_message
+            .chat_id
+            .set_visibility(bob, ChatVisibility::Archived)
+            .await?;
+        SystemTime::shift(Duration::from_secs(100));
+
+        delete_expired_messages(bob, time()).await?;
+
+        assert!(Message::load_from_db_optional(bob, bob_received_message.id)
+            .await?
+            .is_none());
+
+        // Bob mutes the chat so it is not unarchived.
+        set_muted(bob, bob_received_message.chat_id, MuteDuration::Forever).await?;
+
+        // Now test that for already archived chat
+        // timer is started if all archived chats are marked as noticed.
+        let bob_received_message_2 = tcm.send_recv(alice, bob, "Hello again!").await;
+        assert_eq!(bob_received_message_2.state, MessageState::InFresh);
+
+        marknoticed_chat(bob, DC_CHAT_ID_ARCHIVED_LINK).await?;
+        SystemTime::shift(Duration::from_secs(100));
+
+        delete_expired_messages(bob, time()).await?;
+
+        assert!(
+            Message::load_from_db_optional(bob, bob_received_message_2.id)
+                .await?
+                .is_none()
+        );
+
+        Ok(())
+    }
 }

src/events/payload.rs

@@ -59,7 +59,7 @@ pub enum EventType {
     /// or for functions that are expected to fail (eg. dc_continue_key_transfer())
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
-    /// in a messasge box then.
+    /// in a message box then.
     Error(String),
 
     /// An action cannot be performed because the user is not in the group.
@@ -107,6 +107,24 @@ pub enum EventType {
         reaction: Reaction,
     },
 
+    /// A webxdc wants an info message or a changed summary to be notified.
+    IncomingWebxdcNotify {
+        /// ID of the chat.
+        chat_id: ChatId,
+
+        /// ID of the contact sending.
+        contact_id: ContactId,
+
+        /// ID of the added info message or webxdc instance in case of summary change.
+        msg_id: MsgId,
+
+        /// Text to notify.
+        text: String,
+
+        /// Link assigned to this notification, if any.
+        href: Option<String>,
+    },
+
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
@@ -332,6 +350,20 @@ pub enum EventType {
         chat_id: Option<ChatId>,
     },
 
+    /// Inform that the list of accounts has changed (an account removed or added or (not yet implemented) the account order changes)
+    ///
+    /// This event is only emitted by the account manager
+    AccountsChanged,
+
+    /// Inform that an account property that might be shown in the account list changed, namely:
+    /// - is_configured (see [crate::context::Context::is_configured])
+    /// - displayname
+    /// - selfavatar
+    /// - private_tag
+    ///
+    /// This event is emitted from the account whose property changed.
+    AccountsItemChanged,
+
     /// Event for using in tests, e.g. as a fence between normally generated events.
     #[cfg(test)]
     Test,

src/headerdef.rs

@@ -73,6 +73,7 @@ pub enum HeaderDef {
 
     /// [Autocrypt](https://autocrypt.org/) header.
     Autocrypt,
+    AutocryptGossip,
     AutocryptSetupMessage,
     SecureJoin,
 

src/html.rs

@@ -7,6 +7,8 @@
 //! `MsgId.get_html()` will return HTML -
 //! this allows nice quoting, handling linebreaks properly etc.
 
+use std::mem;
+
 use anyhow::{Context as _, Result};
 use base64::Engine as _;
 use lettre_email::mime::Mime;
@@ -77,21 +79,26 @@ fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
 struct HtmlMsgParser {
     pub html: String,
     pub plain: Option<PlainText>,
+    pub(crate) msg_html: String,
 }
 
 impl HtmlMsgParser {
     /// Function takes a raw mime-message string,
     /// searches for the main-text part
     /// and returns that as parser.html
-    pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
+    pub async fn from_bytes<'a>(
+        context: &Context,
+        rawmime: &'a [u8],
+    ) -> Result<(Self, mailparse::ParsedMail<'a>)> {
         let mut parser = HtmlMsgParser {
             html: "".to_string(),
             plain: None,
+            msg_html: "".to_string(),
         };
 
-        let parsedmail = mailparse::parse_mail(rawmime)?;
+        let parsedmail = mailparse::parse_mail(rawmime).context("Failed to parse mail")?;
 
-        parser.collect_texts_recursive(&parsedmail).await?;
+        parser.collect_texts_recursive(context, &parsedmail).await?;
 
         if parser.html.is_empty() {
             if let Some(plain) = &parser.plain {
@@ -100,8 +107,8 @@ impl HtmlMsgParser {
         } else {
             parser.cid_to_data_recursive(context, &parsedmail).await?;
         }
-
+        parser.html += &mem::take(&mut parser.msg_html);
-        Ok(parser)
+        Ok((parser, parsedmail))
     }
 
     /// Function iterates over all mime-parts
@@ -114,12 +121,13 @@ impl HtmlMsgParser {
     /// therefore we use the first one.
     async fn collect_texts_recursive<'a>(
         &'a mut self,
+        context: &'a Context,
         mail: &'a mailparse::ParsedMail<'a>,
     ) -> Result<()> {
         match get_mime_multipart_type(&mail.ctype) {
             MimeMultipartType::Multiple => {
                 for cur_data in &mail.subparts {
-                    Box::pin(self.collect_texts_recursive(cur_data)).await?
+                    Box::pin(self.collect_texts_recursive(context, cur_data)).await?
                 }
                 Ok(())
             }
@@ -128,8 +136,35 @@ impl HtmlMsgParser {
                 if raw.is_empty() {
                     return Ok(());
                 }
-                let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?;
+                let (parser, mail) = Box::pin(HtmlMsgParser::from_bytes(context, &raw)).await?;
-                Box::pin(self.collect_texts_recursive(&mail)).await
+                if !parser.html.is_empty() {
+                    let mut text = "\r\n\r\n".to_string();
+                    for h in mail.headers {
+                        let key = h.get_key();
+                        if matches!(
+                            key.to_lowercase().as_str(),
+                            "date"
+                                | "from"
+                                | "sender"
+                                | "reply-to"
+                                | "to"
+                                | "cc"
+                                | "bcc"
+                                | "subject"
+                        ) {
+                            text += &format!("{key}: {}\r\n", h.get_value());
+                        }
+                    }
+                    text += "\r\n";
+                    self.msg_html += &PlainText {
+                        text,
+                        flowed: false,
+                        delsp: false,
+                    }
+                    .to_html();
+                    self.msg_html += &parser.html;
+                }
+                Ok(())
             }
             MimeMultipartType::Single => {
                 let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
@@ -144,12 +179,12 @@ impl HtmlMsgParser {
                         self.plain = Some(PlainText {
                             text: decoded_data,
                             flowed: if let Some(format) = mail.ctype.params.get("format") {
-                                format.as_str().to_ascii_lowercase() == "flowed"
+                                format.as_str().eq_ignore_ascii_case("flowed")
                             } else {
                                 false
                             },
                             delsp: if let Some(delsp) = mail.ctype.params.get("delsp") {
-                                delsp.as_str().to_ascii_lowercase() == "yes"
+                                delsp.as_str().eq_ignore_ascii_case("yes")
                             } else {
                                 false
                             },
@@ -175,14 +210,7 @@ impl HtmlMsgParser {
                 }
                 Ok(())
             }
-            MimeMultipartType::Message => {
+            MimeMultipartType::Message => Ok(()),
-                let raw = mail.get_body_raw()?;
-                if raw.is_empty() {
-                    return Ok(());
-                }
-                let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?;
-                Box::pin(self.cid_to_data_recursive(context, &mail)).await
-            }
             MimeMultipartType::Single => {
                 let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
                 if mimetype.type_() == mime::IMAGE {
@@ -240,7 +268,7 @@ impl MsgId {
                     warn!(context, "get_html: parser error: {:#}", err);
                     Ok(None)
                 }
-                Ok(parser) => Ok(Some(parser.html)),
+                Ok((parser, _)) => Ok(Some(parser.html)),
             }
         } else {
             warn!(context, "get_html: no mime for {}", self);
@@ -274,7 +302,7 @@ mod tests {
     async fn test_htmlparse_plain_unspecified() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
             parser.html,
             r#"<!DOCTYPE html>
@@ -283,7 +311,6 @@ mod tests {
 <meta name="color-scheme" content="light dark" />
 </head><body>
 This message does not have Content-Type nor Subject.<br/>
-<br/>
 </body></html>
 "#
         );
@@ -293,7 +320,7 @@ This message does not have Content-Type nor Subject.<br/>
     async fn test_htmlparse_plain_iso88591() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
             parser.html,
             r#"<!DOCTYPE html>
@@ -302,7 +329,6 @@ This message does not have Content-Type nor Subject.<br/>
 <meta name="color-scheme" content="light dark" />
 </head><body>
 message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
-<br/>
 </body></html>
 "#
         );
@@ -312,7 +338,7 @@ message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
     async fn test_htmlparse_plain_flowed() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(parser.plain.unwrap().flowed);
         assert_eq!(
             parser.html,
@@ -325,7 +351,6 @@ This line ends with a space and will be merged with the next one due to format=f
 <br/>
 This line does not end with a space<br/>
 and will be wrapped as usual.<br/>
-<br/>
 </body></html>
 "#
         );
@@ -335,7 +360,7 @@ and will be wrapped as usual.<br/>
     async fn test_htmlparse_alt_plain() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
             parser.html,
             r#"<!DOCTYPE html>
@@ -347,7 +372,6 @@ mime-modified should not be set set as there is no html and no special stuff;<br
 although not being a delta-message.<br/>
 test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x27; :)<br/>
 <br/>
-<br/>
 </body></html>
 "#
         );
@@ -357,7 +381,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
     async fn test_htmlparse_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_html.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
 
         // on windows, `\r\n` linends are returned from mimeparser,
         // however, rust multiline-strings use just `\n`;
@@ -375,7 +399,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
     async fn test_htmlparse_alt_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_html.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
             parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
@@ -390,7 +414,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
     async fn test_htmlparse_alt_plain_html() {
         let t = TestContext::new().await;
         let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert_eq!(
             parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
             r##"<html>
@@ -415,7 +439,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
         assert!(test.find("data:").is_none());
 
         // parsing converts cid: to data:
-        let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
+        let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
         assert!(parser.html.contains("<html>"));
         assert!(!parser.html.contains("Content-Id:"));
         assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ"));
@@ -525,8 +549,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
 
         // alice sends a message with html-part to bob
         let chat_id = alice.create_chat(&bob).await.id;
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("plain text".to_string());
-        msg.set_text("plain text".to_string());
         msg.set_html(Some("<b>html</b> text".to_string()));
         assert!(msg.mime_modified);
         chat::send_msg(&alice, chat_id, &mut msg).await.unwrap();

src/imap.rs

@@ -13,7 +13,7 @@ use std::{
     time::{Duration, UNIX_EPOCH},
 };
 
-use anyhow::{bail, format_err, Context as _, Result};
+use anyhow::{bail, ensure, format_err, Context as _, Result};
 use async_channel::Receiver;
 use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
 use deltachat_contact_tools::ContactAddress;
@@ -36,16 +36,16 @@ use crate::log::LogExt;
 use crate::login_param::{
     prioritize_server_login_params, ConfiguredLoginParam, ConfiguredServerLoginParam,
 };
-use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
+use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
 use crate::mimeparser;
 use crate::net::proxy::ProxyConfig;
 use crate::net::session::SessionStream;
 use crate::oauth2::get_oauth2_access_token;
+use crate::push::encrypt_device_token;
 use crate::receive_imf::{
     from_field_to_contact_id, get_prefetch_parent_message, receive_imf_inner, ReceivedMsg,
 };
 use crate::scheduler::connectivity::ConnectivityStore;
-use crate::sql;
 use crate::stock_str;
 use crate::tools::{self, create_id, duration_to_str};
 
@@ -89,7 +89,7 @@ pub(crate) struct Imap {
 
     oauth2: bool,
 
-    login_failed_once: bool,
+    authentication_failed_once: bool,
 
     pub(crate) connectivity: ConnectivityStore,
 
@@ -254,7 +254,7 @@ impl Imap {
             proxy_config,
             strict_tls,
             oauth2,
-            login_failed_once: false,
+            authentication_failed_once: false,
             connectivity: Default::default(),
             conn_last_try: UNIX_EPOCH,
             conn_backoff_ms: 0,
@@ -290,7 +290,11 @@ impl Imap {
     /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
     /// instead if you are going to actually use connection rather than trying connection
     /// parameters.
-    pub(crate) async fn connect(&mut self, context: &Context) -> Result<Session> {
+    pub(crate) async fn connect(
+        &mut self,
+        context: &Context,
+        configuring: bool,
+    ) -> Result<Session> {
         let now = tools::Time::now();
         let until_can_send = max(
             min(self.conn_last_try, now)
@@ -398,12 +402,12 @@ impl Imap {
                     let mut lock = context.server_id.write().await;
                     lock.clone_from(&session.capabilities.server_id);
 
-                    self.login_failed_once = false;
+                    self.authentication_failed_once = false;
                     context.emit_event(EventType::ImapConnected(format!(
                         "IMAP-LOGIN as {}",
                         lp.user
                     )));
-                    self.connectivity.set_connected(context).await;
+                    self.connectivity.set_preparing(context).await;
                     info!(context, "Successfully logged into IMAP server");
                     return Ok(session);
                 }
@@ -412,25 +416,17 @@ impl Imap {
                     let imap_user = lp.user.to_owned();
                     let message = stock_str::cannot_login(context, &imap_user).await;
 
-                    let err_str = err.to_string();
                     warn!(context, "IMAP failed to login: {err:#}.");
                     first_error.get_or_insert(format_err!("{message} ({err:#})"));
 
-                    let lock = context.wrong_pw_warning_mutex.lock().await;
+                    // If it looks like the password is wrong, send a notification:
-                    if self.login_failed_once
+                    let _lock = context.wrong_pw_warning_mutex.lock().await;
-                        && err_str.to_lowercase().contains("authentication")
+                    if err.to_string().to_lowercase().contains("authentication") {
+                        if self.authentication_failed_once
+                            && !configuring
                             && context.get_config_bool(Config::NotifyAboutWrongPw).await?
                         {
-                        if let Err(e) = context
+                            let mut msg = Message::new_text(message);
-                            .set_config_internal(Config::NotifyAboutWrongPw, None)
-                            .await
-                        {
-                            warn!(context, "{e:#}.");
-                        }
-                        drop(lock);
-
-                        let mut msg = Message::new(Viewtype::Text);
-                        msg.text.clone_from(&message);
                             if let Err(e) = chat::add_device_msg_with_importance(
                                 context,
                                 None,
@@ -440,9 +436,18 @@ impl Imap {
                             .await
                             {
                                 warn!(context, "Failed to add device message: {e:#}.");
+                            } else {
+                                context
+                                    .set_config_internal(Config::NotifyAboutWrongPw, None)
+                                    .await
+                                    .log_err(context)
+                                    .ok();
                             }
                         } else {
-                        self.login_failed_once = true;
+                            self.authentication_failed_once = true;
+                        }
+                    } else {
+                        self.authentication_failed_once = false;
                     }
                 }
             }
@@ -456,7 +461,8 @@ impl Imap {
     /// Ensure that IMAP client is connected, folders are created and IMAP capabilities are
     /// determined.
     pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
-        let mut session = match self.connect(context).await {
+        let configuring = false;
+        let mut session = match self.connect(context, configuring).await {
             Ok(session) => session,
             Err(err) => {
                 self.connectivity.set_err(context, &err).await;
@@ -534,10 +540,14 @@ impl Imap {
             return Ok(false);
         }
 
-        session
+        let create = false;
-            .select_with_uidvalidity(context, folder)
+        let folder_exists = session
+            .select_with_uidvalidity(context, folder, create)
             .await
             .with_context(|| format!("Failed to select folder {folder:?}"))?;
+        if !folder_exists {
+            return Ok(false);
+        }
 
         if !session.new_mail && !fetch_existing_msgs {
             info!(context, "No new emails in folder {folder:?}.");
@@ -829,15 +839,19 @@ impl Session {
         folder: &str,
         folder_meaning: FolderMeaning,
     ) -> Result<()> {
+        let uid_validity;
         // Collect pairs of UID and Message-ID.
         let mut msgs = BTreeMap::new();
 
-        self.select_with_uidvalidity(context, folder).await?;
+        let create = false;
-
+        let folder_exists = self
+            .select_with_uidvalidity(context, folder, create)
+            .await?;
+        if folder_exists {
             let mut list = self
                 .uid_fetch("1:*", RFC724MID_UID)
                 .await
-            .with_context(|| format!("can't resync folder {folder}"))?;
+                .with_context(|| format!("Can't resync folder {folder}"))?;
             while let Some(fetch) = list.try_next().await? {
                 let headers = match get_fetch_headers(&fetch) {
                     Ok(headers) => headers,
@@ -861,12 +875,15 @@ impl Session {
 
             info!(
                 context,
-            "Resync: collected {} message IDs in folder {}",
+                "resync_folder_uids: Collected {} message IDs in {folder}.",
                 msgs.len(),
-            folder,
             );
 
-        let uid_validity = get_uidvalidity(context, folder).await?;
+            uid_validity = get_uidvalidity(context, folder).await?;
+        } else {
+            warn!(context, "resync_folder_uids: No folder {folder}.");
+            uid_validity = 0;
+        }
 
         // Write collected UIDs to SQLite database.
         context
@@ -904,15 +921,15 @@ impl Session {
             .await?;
         context
             .sql
-            .execute(
+            .transaction(|transaction| {
-                &format!(
+                let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
-                    "DELETE FROM imap WHERE id IN ({})",
+                for row_id in row_ids {
-                    sql::repeat_vars(row_ids.len())
+                    stmt.execute((row_id,))?;
-                ),
+                }
-                rusqlite::params_from_iter(row_ids),
+                Ok(())
-            )
+            })
             .await
-            .context("cannot remove deleted messages from imap table")?;
+            .context("Cannot remove deleted messages from imap table")?;
 
         context.emit_event(EventType::ImapMessageDeleted(format!(
             "IMAP messages {uid_set} marked as deleted"
@@ -935,15 +952,15 @@ impl Session {
                     // Messages are moved or don't exist, IMAP returns OK response in both cases.
                     context
                         .sql
-                        .execute(
+                        .transaction(|transaction| {
-                            &format!(
+                            let mut stmt = transaction.prepare("DELETE FROM imap WHERE id = ?")?;
-                                "DELETE FROM imap WHERE id IN ({})",
+                            for row_id in row_ids {
-                                sql::repeat_vars(row_ids.len())
+                                stmt.execute((row_id,))?;
-                            ),
+                            }
-                            rusqlite::params_from_iter(row_ids),
+                            Ok(())
-                        )
+                        })
                         .await
-                        .context("cannot delete moved messages from imap table")?;
+                        .context("Cannot delete moved messages from imap table")?;
                     context.emit_event(EventType::ImapMessageMoved(format!(
                         "IMAP messages {set} moved to {target}"
                     )));
@@ -989,15 +1006,15 @@ impl Session {
         }
         context
             .sql
-            .execute(
+            .transaction(|transaction| {
-                &format!(
+                let mut stmt = transaction.prepare("UPDATE imap SET target='' WHERE id = ?")?;
-                    "UPDATE imap SET target='' WHERE id IN ({})",
+                for row_id in row_ids {
-                    sql::repeat_vars(row_ids.len())
+                    stmt.execute((row_id,))?;
-                ),
+                }
-                rusqlite::params_from_iter(row_ids),
+                Ok(())
-            )
+            })
             .await
-            .context("cannot plan deletion of messages")?;
+            .context("Cannot plan deletion of messages")?;
         if copy {
             context.emit_event(EventType::ImapMessageMoved(format!(
                 "IMAP messages {set} copied to {target}"
@@ -1033,7 +1050,11 @@ impl Session {
             // MOVE/DELETE operations. This does not result in multiple SELECT commands
             // being sent because `select_folder()` does nothing if the folder is already
             // selected.
-            self.select_with_uidvalidity(context, folder).await?;
+            let create = false;
+            let folder_exists = self
+                .select_with_uidvalidity(context, folder, create)
+                .await?;
+            ensure!(folder_exists, "No folder {folder}");
 
             // Empty target folder name means messages should be deleted.
             if target.is_empty() {
@@ -1127,29 +1148,40 @@ impl Session {
             .await?;
 
         for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
-            if let Err(err) = self.select_with_uidvalidity(context, &folder).await {
+            let create = false;
-                warn!(context, "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}.");
+            let folder_exists = match self.select_with_uidvalidity(context, &folder, create).await {
+                Err(err) => {
+                    warn!(
+                        context,
+                        "store_seen_flags_on_imap: Failed to select {folder}, will retry later: {err:#}.");
+                    continue;
+                }
+                Ok(folder_exists) => folder_exists,
+            };
+            if !folder_exists {
+                warn!(context, "store_seen_flags_on_imap: No folder {folder}.");
             } else if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
                 warn!(
                     context,
                     "Cannot mark messages {uid_set} in {folder} as seen, will retry later: {err:#}.");
+                continue;
             } else {
                 info!(
                     context,
                     "Marked messages {} in folder {} as seen.", uid_set, folder
                 );
+            }
             context
                 .sql
-                    .execute(
+                .transaction(|transaction| {
-                        &format!(
+                    let mut stmt = transaction.prepare("DELETE FROM imap_markseen WHERE id = ?")?;
-                            "DELETE FROM imap_markseen WHERE id IN ({})",
+                    for rowid in rowid_set {
-                            sql::repeat_vars(rowid_set.len())
+                        stmt.execute((rowid,))?;
-                        ),
-                        rusqlite::params_from_iter(rowid_set),
-                    )
-                    .await
-                    .context("cannot remove messages marked as seen from imap_markseen table")?;
                     }
+                    Ok(())
+                })
+                .await
+                .context("Cannot remove messages marked as seen from imap_markseen table")?;
         }
 
         Ok(())
@@ -1165,9 +1197,14 @@ impl Session {
             return Ok(());
         }
 
-        self.select_with_uidvalidity(context, folder)
+        let create = false;
+        let folder_exists = self
+            .select_with_uidvalidity(context, folder, create)
             .await
-            .context("failed to select folder")?;
+            .context("Failed to select folder")?;
+        if !folder_exists {
+            return Ok(());
+        }
 
         let mailbox = self
             .selected_mailbox
@@ -1196,6 +1233,8 @@ impl Session {
             .await
             .context("failed to fetch flags")?;
 
+        let mut got_unsolicited_fetch = false;
+
         while let Some(fetch) = list
             .try_next()
             .await
@@ -1205,6 +1244,7 @@ impl Session {
                 uid
             } else {
                 info!(context, "FETCH result contains no UID, skipping");
+                got_unsolicited_fetch = true;
                 continue;
             };
             let is_seen = fetch.flags().any(|flag| flag == Flag::Seen);
@@ -1227,6 +1267,15 @@ impl Session {
                 warn!(context, "FETCH result contains no MODSEQ");
             }
         }
+        drop(list);
+
+        if got_unsolicited_fetch {
+            // We got unsolicited FETCH, which means some flags
+            // have been modified while our request was in progress.
+            // We may or may not have these new flags as a part of the response,
+            // so better skip next IDLE and do another round of flag synchronization.
+            self.new_mail = true;
+        }
 
         set_modseq(context, folder, highest_modseq)
             .await
@@ -1542,16 +1591,54 @@ impl Session {
         };
 
         if self.can_metadata() && self.can_push() {
+            let device_token_changed = context
+                .get_config(Config::DeviceToken)
+                .await?
+                .map_or(true, |config_token| device_token != config_token);
+
+            if device_token_changed {
                 let folder = context
                     .get_config(Config::ConfiguredInboxFolder)
                     .await?
                     .context("INBOX is not configured")?;
 
-            self.run_command_and_check_ok(format!(
+                let encrypted_device_token = encrypt_device_token(&device_token)
-                "SETMETADATA \"{folder}\" (/private/devicetoken \"{device_token}\")"
+                    .context("Failed to encrypt device token")?;
+
+                // We expect that the server supporting `XDELTAPUSH` capability
+                // has non-synchronizing literals support as well:
+                // <https://www.rfc-editor.org/rfc/rfc7888>.
+                let encrypted_device_token_len = encrypted_device_token.len();
+
+                if encrypted_device_token_len <= 4096 {
+                    self.run_command_and_check_ok(&format_setmetadata(
+                        &folder,
+                        &encrypted_device_token,
                     ))
                     .await
                     .context("SETMETADATA command failed")?;
+
+                    // Store device token saved on the server
+                    // to prevent storing duplicate tokens.
+                    // The server cannot deduplicate on its own
+                    // because encryption gives a different
+                    // result each time.
+                    context
+                        .set_config_internal(Config::DeviceToken, Some(&device_token))
+                        .await?;
+                } else {
+                    // If Apple or Google (FCM) gives us a very large token,
+                    // do not even try to give it to IMAP servers.
+                    //
+                    // Limit of 4096 is arbitrarily selected
+                    // to be the same as required by LITERAL- IMAP extension.
+                    //
+                    // Dovecot supports LITERAL+ and non-synchronizing literals
+                    // of any length, but there is no reason for tokens
+                    // to be that large even after OpenPGP encryption.
+                    warn!(context, "Device token is too long for LITERAL-, ignoring.");
+                }
+            }
             context.push_subscribed.store(true, Ordering::Relaxed);
         } else if !context.push_subscriber.heartbeat_subscribed().await {
             let context = context.clone();
@@ -1563,10 +1650,17 @@ impl Session {
     }
 }
 
+fn format_setmetadata(folder: &str, device_token: &str) -> String {
+    let device_token_len = device_token.len();
+    format!(
+        "SETMETADATA \"{folder}\" (/private/devicetoken {{{device_token_len}+}}\r\n{device_token})"
+    )
+}
+
 impl Session {
     /// Returns success if we successfully set the flag or we otherwise
     /// think add_flag should not be retried: Disconnection during setting
-    /// the flag, or other imap-errors, returns true as well.
+    /// the flag, or other imap-errors, returns Ok as well.
     ///
     /// Returning error means that the operation can be retried.
     async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
@@ -1613,7 +1707,11 @@ impl Session {
                 self.close().await?;
                 // Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
                 // emails moved before that wouldn't be fetched but considered "old" instead.
-                self.select_with_uidvalidity(context, folder).await?;
+                let create = false;
+                let folder_exists = self
+                    .select_with_uidvalidity(context, folder, create)
+                    .await?;
+                ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
                 return Ok(Some(folder));
             }
         }
@@ -1624,7 +1722,10 @@ impl Session {
         // Some servers require namespace-style folder names like "INBOX.DeltaChat", so we try all
         // the variants here.
         for folder in folders {
-            match self.select_with_uidvalidity(context, folder).await {
+            match self
+                .select_with_uidvalidity(context, folder, create_mvbox)
+                .await
+            {
                 Ok(_) => {
                     info!(context, "MVBOX-folder {} created.", folder);
                     return Ok(Some(folder));
@@ -1712,17 +1813,21 @@ impl Imap {
 }
 
 impl Session {
-    /// Return whether the server sent an unsolicited EXISTS response.
+    /// Return whether the server sent an unsolicited EXISTS or FETCH response.
+    ///
     /// Drains all responses from `session.unsolicited_responses` in the process.
-    /// If this returns `true`, this means that new emails arrived and you should
+    ///
-    /// fetch again, even if you just fetched.
+    /// If this returns `true`, this means that new emails arrived
-    fn server_sent_unsolicited_exists(&self, context: &Context) -> Result<bool> {
+    /// or flags have been changed.
+    /// In this case we may want to skip next IDLE and do a round
+    /// of fetching new messages and synchronizing seen flags.
+    fn drain_unsolicited_responses(&self, context: &Context) -> Result<bool> {
         use async_imap::imap_proto::Response;
         use async_imap::imap_proto::ResponseCode;
         use UnsolicitedResponse::*;
 
         let folder = self.selected_folder.as_deref().unwrap_or_default();
-        let mut unsolicited_exists = false;
+        let mut should_refetch = false;
         while let Ok(response) = self.unsolicited_responses.try_recv() {
             match response {
                 Exists(_) => {
@@ -1730,28 +1835,38 @@ impl Session {
                         context,
                         "Need to refetch {folder:?}, got unsolicited EXISTS {response:?}"
                     );
-                    unsolicited_exists = true;
+                    should_refetch = true;
+                }
+
+                Expunge(_) | Recent(_) => {}
+                Other(ref response_data) => {
+                    match response_data.parsed() {
+                        Response::Fetch { .. } => {
+                            info!(
+                                context,
+                                "Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
+                            );
+                            should_refetch = true;
                         }
 
                         // We are not interested in the following responses and they are are
-                // sent quite frequently, so, we ignore them without logging them
+                        // sent quite frequently, so, we ignore them without logging them.
-                Expunge(_) | Recent(_) => {}
+                        Response::Done {
-                Other(response_data)
-                    if matches!(
-                        response_data.parsed(),
-                        Response::Fetch { .. }
-                            | Response::Done {
                             code: Some(ResponseCode::CopyUid(_, _, _)),
                             ..
-                            }
+                        } => {}
-                    ) => {}
 
                         _ => {
                             info!(context, "{folder:?}: got unsolicited response {response:?}")
                         }
                     }
                 }
-        Ok(unsolicited_exists)
+                _ => {
+                    info!(context, "{folder:?}: got unsolicited response {response:?}")
+                }
+            }
+        }
+        Ok(should_refetch)
     }
 }
 
@@ -1884,7 +1999,7 @@ async fn needs_move_to_mvbox(
         && has_chat_version
         && headers
             .get_header_value(HeaderDef::AutoSubmitted)
-            .filter(|val| val.to_ascii_lowercase() == "auto-generated")
+            .filter(|val| val.eq_ignore_ascii_case("auto-generated"))
             .is_some()
     {
         if let Some(from) = mimeparser::get_from(headers) {
@@ -2461,10 +2576,14 @@ async fn add_all_recipients_as_contacts(
         );
         return Ok(());
     };
-    session
+    let create = false;
-        .select_with_uidvalidity(context, &mailbox)
+    let folder_exists = session
+        .select_with_uidvalidity(context, &mailbox, create)
         .await
         .with_context(|| format!("could not select {mailbox}"))?;
+    if !folder_exists {
+        return Ok(());
+    }
 
     let recipients = session
         .get_all_recipients(context)
@@ -2832,4 +2951,16 @@ mod tests {
             vec![("INBOX".to_string(), vec![1, 2, 3], "2:3".to_string())]
         );
     }
+
+    #[test]
+    fn test_setmetadata_device_token() {
+        assert_eq!(
+            format_setmetadata("INBOX", "foobarbaz"),
+            "SETMETADATA \"INBOX\" (/private/devicetoken {9+}\r\nfoobarbaz)"
+        );
+        assert_eq!(
+            format_setmetadata("INBOX", "foo\r\nbar\r\nbaz\r\n"),
+            "SETMETADATA \"INBOX\" (/private/devicetoken {15+}\r\nfoo\r\nbar\r\nbaz\r\n)"
+        );
+    }
 }

src/imap/idle.rs

@@ -9,7 +9,6 @@ use tokio::time::timeout;
 use super::session::Session;
 use super::Imap;
 use crate::context::Context;
-use crate::imap::FolderMeaning;
 use crate::net::TIMEOUT;
 use crate::tools::{self, time_elapsed};
 
@@ -30,9 +29,11 @@ impl Session {
     ) -> Result<Self> {
         use futures::future::FutureExt;
 
-        self.select_with_uidvalidity(context, folder).await?;
+        let create = true;
+        self.select_with_uidvalidity(context, folder, create)
+            .await?;
 
-        if self.server_sent_unsolicited_exists(context)? {
+        if self.drain_unsolicited_responses(context)? {
             self.new_mail = true;
         }
 
@@ -109,37 +110,16 @@ impl Imap {
     pub(crate) async fn fake_idle(
         &mut self,
         context: &Context,
-        session: &mut Session,
         watch_folder: String,
-        folder_meaning: FolderMeaning,
     ) -> Result<()> {
         let fake_idle_start_time = tools::Time::now();
 
         info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-        // Loop until we are interrupted or until we fetch something.
+        // Wait for 60 seconds or until we are interrupted.
-        loop {
         match timeout(Duration::from_secs(60), self.idle_interrupt_receiver.recv()).await {
-                Err(_) => {
+            Err(_) => info!(context, "Fake IDLE finished."),
-                    // Let's see if fetching messages results
+            Ok(_) => info!(context, "Fake IDLE interrupted."),
-                    // in anything.  If so, we behave as if IDLE had data but
-                    // will have already fetched the messages so perform_*_fetch
-                    // will not find any new.
-                    let res = self
-                        .fetch_new_messages(context, session, &watch_folder, folder_meaning, false)
-                        .await?;
-
-                    info!(context, "fetch_new_messages returned {:?}", res);
-
-                    if res {
-                        break;
-                    }
-                }
-                Ok(_) => {
-                    info!(context, "Fake IDLE interrupted.");
-                    break;
-                }
-            }
         }
 
         info!(

src/imap/scan_folders.rs

@@ -34,6 +34,7 @@ impl Imap {
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
+        let mut folder_names = Vec::new();
 
         for folder in folders {
             let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
@@ -44,6 +45,7 @@ impl Imap {
                 // already been moved and left it in the inbox.
                 continue;
             }
+            folder_names.push(folder.name().to_string());
             let folder_name_meaning = get_folder_meaning_by_name(folder.name());
 
             if let Some(config) = folder_meaning.to_config() {
@@ -66,21 +68,11 @@ impl Imap {
                 && folder_meaning != FolderMeaning::Drafts
                 && folder_meaning != FolderMeaning::Trash
             {
-                // Drain leftover unsolicited EXISTS messages
-                session.server_sent_unsolicited_exists(context)?;
-
-                loop {
                 self.fetch_move_delete(context, session, folder.name(), folder_meaning)
                     .await
                     .context("Can't fetch new msgs in scanned folder")
                     .log_err(context)
                     .ok();
-
-                    // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
-                    if !session.server_sent_unsolicited_exists(context)? {
-                        break;
-                    }
-                }
             }
         }
 
@@ -101,6 +93,7 @@ impl Imap {
             }
         }
 
+        info!(context, "Found folders: {folder_names:?}.");
         last_scan.replace(tools::Time::now());
         Ok(true)
     }

src/imap/select_folder.rs

@@ -111,7 +111,8 @@ impl ImapSession {
         }
     }
 
-    /// Selects a folder and takes care of UIDVALIDITY changes.
+    /// Selects a folder optionally creating it and takes care of UIDVALIDITY changes. Returns false
+    /// iff `folder` doesn't exist.
     ///
     /// When selecting a folder for the first time, sets the uid_next to the current
     /// mailbox.uid_next so that no old emails are fetched.
@@ -123,11 +124,24 @@ impl ImapSession {
         &mut self,
         context: &Context,
         folder: &str,
-    ) -> Result<()> {
+        create: bool,
-        let newly_selected = self
+    ) -> Result<bool> {
-            .select_or_create_folder(context, folder)
+        let newly_selected = if create {
+            self.select_or_create_folder(context, folder)
                 .await
-            .with_context(|| format!("failed to select or create folder {folder}"))?;
+                .with_context(|| format!("failed to select or create folder {folder}"))?
+        } else {
+            match self.select_folder(context, folder).await {
+                Ok(newly_selected) => newly_selected,
+                Err(err) => match err {
+                    Error::NoFolder(..) => return Ok(false),
+                    _ => {
+                        return Err(err)
+                            .with_context(|| format!("failed to select folder {folder}"))?
+                    }
+                },
+            }
+        };
         let mailbox = self
             .selected_mailbox
             .as_mut()
@@ -199,7 +213,7 @@ impl ImapSession {
                 }
             }
 
-            return Ok(());
+            return Ok(true);
         }
 
         // UIDVALIDITY is modified, reset highest seen MODSEQ.
@@ -233,7 +247,7 @@ impl ImapSession {
             old_uid_next,
             old_uid_validity,
         );
-        Ok(())
+        Ok(true)
     }
 }
 

src/imex.rs

@@ -4,7 +4,7 @@ use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
 use std::pin::Pin;
 
-use ::pgp::types::KeyTrait;
+use ::pgp::types::PublicKeyTrait;
 use anyhow::{bail, ensure, format_err, Context as _, Result};
 use futures::TryStreamExt;
 use futures_lite::FutureExt;
@@ -416,7 +416,7 @@ async fn import_backup_stream_inner<R: tokio::io::AsyncRead + Unpin>(
             .context("cannot import unpacked database");
     }
     if res.is_ok() {
-        res = adjust_delete_server_after(context).await;
+        res = adjust_bcc_self(context).await;
     }
     fs::remove_file(unpacked_database)
         .await
@@ -426,6 +426,7 @@ async fn import_backup_stream_inner<R: tokio::io::AsyncRead + Unpin>(
     if res.is_ok() {
         context.emit_event(EventType::ImexProgress(999));
         res = context.sql.run_migrations(context).await;
+        context.emit_event(EventType::AccountsItemChanged);
     }
     if res.is_ok() {
         delete_and_reset_all_device_msgs(context)
@@ -750,7 +751,7 @@ where
             true => "private",
         };
         let id = id.map_or("default".into(), |i| i.to_string());
-        let fp = DcKey::fingerprint(key).hex();
+        let fp = key.dc_fingerprint().hex();
         format!("{kind}-key-{addr}-{id}-{fp}.asc")
     };
     let path = dir.join(&file_name);
@@ -795,7 +796,7 @@ async fn export_database(
         .to_str()
         .with_context(|| format!("path {} is not valid unicode", dest.display()))?;
 
-    adjust_delete_server_after(context).await?;
+    adjust_bcc_self(context).await?;
     context
         .sql
         .set_raw_config_int("backup_time", timestamp)
@@ -825,15 +826,14 @@ async fn export_database(
         .await
 }
 
-/// Sets `Config::DeleteServerAfter` to "never" if needed so that new messages are present on the
+/// Sets `Config::BccSelf` (and `DeleteServerAfter` to "never" in effect) if needed so that new
-/// server after a backup restoration or available for all devices in multi-device case.
+/// messages are present on the server after a backup restoration or available for all devices in
-/// NB: Calling this after a backup import isn't reliable as we can crash in between, but this is a
+/// multi-device case. NB: Calling this after a backup import isn't reliable as we can crash in
-/// problem only for old backups, new backups already have `DeleteServerAfter` set if necessary.
+/// between, but this is a problem only for old backups, new backups already have `BccSelf` set if
-async fn adjust_delete_server_after(context: &Context) -> Result<()> {
+/// necessary.
-    if context.is_chatmail().await? && !context.config_exists(Config::DeleteServerAfter).await? {
+async fn adjust_bcc_self(context: &Context) -> Result<()> {
-        context
+    if context.is_chatmail().await? && !context.config_exists(Config::BccSelf).await? {
-            .set_config(Config::DeleteServerAfter, Some("0"))
+        context.set_config(Config::BccSelf, Some("1")).await?;
-            .await?;
     }
     Ok(())
 }
@@ -878,7 +878,7 @@ mod tests {
             .unwrap()
             .strip_suffix(".asc")
             .unwrap();
-        assert_eq!(fingerprint, DcKey::fingerprint(&key).hex());
+        assert_eq!(fingerprint, key.dc_fingerprint().hex());
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{blobdir}/{filename}");
         let bytes = tokio::fs::read(&filename).await.unwrap();
@@ -1029,12 +1029,20 @@ mod tests {
 
         let context1 = &TestContext::new_alice().await;
 
-        // Check that the setting is displayed correctly.
+        // Check that the settings are displayed correctly.
+        assert_eq!(
+            context1.get_config(Config::BccSelf).await?,
+            Some("1".to_string())
+        );
         assert_eq!(
             context1.get_config(Config::DeleteServerAfter).await?,
             Some("0".to_string())
         );
         context1.set_config_bool(Config::IsChatmail, true).await?;
+        assert_eq!(
+            context1.get_config(Config::BccSelf).await?,
+            Some("0".to_string())
+        );
         assert_eq!(
             context1.get_config(Config::DeleteServerAfter).await?,
             Some("1".to_string())
@@ -1057,6 +1065,10 @@ mod tests {
         assert!(context2.is_configured().await?);
         assert!(context2.is_chatmail().await?);
         for ctx in [context1, context2] {
+            assert_eq!(
+                ctx.get_config(Config::BccSelf).await?,
+                Some("1".to_string())
+            );
             assert_eq!(
                 ctx.get_config(Config::DeleteServerAfter).await?,
                 Some("0".to_string())

src/imex/transfer.rs

@@ -42,7 +42,7 @@ use tokio_util::sync::CancellationToken;
 use crate::chat::add_device_msg;
 use crate::context::Context;
 use crate::imex::BlobDirContents;
-use crate::message::{Message, Viewtype};
+use crate::message::Message;
 use crate::qr::Qr;
 use crate::stock_str::backup_transfer_msg_body;
 use crate::tools::{create_id, time, TempPathGuard};
@@ -178,6 +178,7 @@ impl BackupProvider {
         }
 
         info!(context, "Received valid backup authentication token.");
+        // Emit a nonzero progress so that UIs can display smth like "Transferring...".
         context.emit_event(EventType::ImexProgress(1));
 
         let blobdir = BlobDirContents::new(&context).await?;
@@ -200,8 +201,7 @@ impl BackupProvider {
         info!(context, "Received backup reception acknowledgement.");
         context.emit_event(EventType::ImexProgress(1000));
 
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text(backup_transfer_msg_body(&context).await);
-        msg.text = backup_transfer_msg_body(&context).await;
         add_device_msg(&context, None, Some(&mut msg)).await?;
 
         Ok(())
@@ -310,6 +310,10 @@ pub async fn get_backup2(
     let mut file_size_buf = [0u8; 8];
     recv_stream.read_exact(&mut file_size_buf).await?;
     let file_size = u64::from_be_bytes(file_size_buf);
+    info!(context, "Received backup file size.");
+    // Emit a nonzero progress so that UIs can display smth like "Transferring...".
+    context.emit_event(EventType::ImexProgress(1));
+
     import_backup_stream(context, recv_stream, file_size, passphrase)
         .await
         .context("Failed to import backup from QUIC stream")?;
@@ -369,6 +373,7 @@ mod tests {
     use std::time::Duration;
 
     use crate::chat::{get_chat_msgs, send_msg, ChatItem};
+    use crate::message::Viewtype;
     use crate::test_utils::TestContextManager;
 
     use super::*;
@@ -382,8 +387,7 @@ mod tests {
 
         // Write a message in the self chat
         let self_chat = ctx0.get_self_chat().await;
-        let mut msg = Message::new(Viewtype::Text);
+        let mut msg = Message::new_text("hi there".to_string());
-        msg.set_text("hi there".to_string());
         send_msg(&ctx0, self_chat.id, &mut msg).await.unwrap();
 
         // Send an attachment in the self chat
@@ -435,13 +439,15 @@ mod tests {
         assert!(msg.save_file(&ctx1, &path).await.is_err());
 
         // Check that both received the ImexProgress events.
-        ctx0.evtracker
+        for ctx in [&ctx0, &ctx1] {
-            .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000)))
+            ctx.evtracker
-            .await;
+                .get_matching(|ev| matches!(ev, EventType::ImexProgress(1)))
-        ctx1.evtracker
+                .await;
+            ctx.evtracker
                 .get_matching(|ev| matches!(ev, EventType::ImexProgress(1000)))
                 .await;
         }
+    }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_drop_provider() {

src/key.rs

@@ -11,7 +11,8 @@ use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 use pgp::ser::Serialize;
-use pgp::types::{KeyTrait, SecretKeyTrait};
+use pgp::types::{PublicKeyTrait, SecretKeyTrait};
+use rand::thread_rng;
 use tokio::runtime::Handle;
 
 use crate::config::Config;
@@ -26,10 +27,42 @@ use crate::tools::{self, time_elapsed};
 /// This trait is implemented for rPGP's [SignedPublicKey] and
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
-pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+pub(crate) trait DcKey: Serialize + Deserializable + PublicKeyTrait + Clone {
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self> {
-        Ok(<Self as Deserializable>::from_bytes(Cursor::new(bytes))?)
+        let res = <Self as Deserializable>::from_bytes(Cursor::new(bytes));
+        if let Ok(res) = res {
+            return Ok(res);
+        }
+
+        // Workaround for keys imported using
+        // Delta Chat core < 1.0.0.
+        // Old Delta Chat core had a bug
+        // that resulted in treating CRC24 checksum
+        // as part of the key when reading ASCII Armor.
+        // Some users that started using Delta Chat in 2019
+        // have such corrupted keys with garbage bytes at the end.
+        //
+        // Garbage is at least 3 bytes long
+        // and may be longer due to padding
+        // at the end of the real key data
+        // and importing the key multiple times.
+        //
+        // If removing 10 bytes is not enough,
+        // the key is likely actually corrupted.
+        for garbage_bytes in 3..std::cmp::min(bytes.len(), 10) {
+            let res = <Self as Deserializable>::from_bytes(Cursor::new(
+                bytes
+                    .get(..bytes.len().saturating_sub(garbage_bytes))
+                    .unwrap_or_default(),
+            ));
+            if let Ok(res) = res {
+                return Ok(res);
+            }
+        }
+
+        // Removing garbage bytes did not help, return the error.
+        Ok(res?)
     }
 
     /// Create a key from a base64 string.
@@ -93,8 +126,8 @@ pub(crate) trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     fn to_asc(&self, header: Option<(&str, &str)>) -> String;
 
     /// The fingerprint for the key.
-    fn fingerprint(&self) -> Fingerprint {
+    fn dc_fingerprint(&self) -> Fingerprint {
-        Fingerprint::new(KeyTrait::fingerprint(self))
+        PublicKeyTrait::fingerprint(self).into()
     }
 
     fn is_private() -> bool;
@@ -233,7 +266,8 @@ impl DcSecretKey for SignedSecretKey {
     fn split_public_key(&self) -> Result<SignedPublicKey> {
         self.verify()?;
         let unsigned_pubkey = SecretKeyTrait::public_key(self);
-        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        let mut rng = thread_rng();
+        let signed_pubkey = unsigned_pubkey.sign(&mut rng, self, || "".into())?;
         Ok(signed_pubkey)
     }
 }
@@ -395,6 +429,12 @@ impl Fingerprint {
     }
 }
 
+impl From<pgp::types::Fingerprint> for Fingerprint {
+    fn from(fingerprint: pgp::types::Fingerprint) -> Fingerprint {
+        Self::new(fingerprint.as_bytes().into())
+    }
+}
+
 impl fmt::Debug for Fingerprint {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("Fingerprint")
@@ -557,6 +597,36 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         }
     }
 
+    /// Tests workaround for Delta Chat core < 1.0.0
+    /// which parsed CRC24 at the end of ASCII Armor
+    /// as the part of the key.
+    /// Depending on the alignment and the number of
+    /// `=` characters at the end of the key,
+    /// this resulted in various number of garbage
+    /// octets at the end of the key, starting from 3 octets,
+    /// but possibly 4 or 5 and maybe more octets
+    /// if the key is imported or transferred
+    /// using Autocrypt Setup Message multiple times.
+    #[test]
+    fn test_ignore_trailing_garbage() {
+        // Test several variants of garbage.
+        for garbage in [
+            b"\x02\xfc\xaa\x38\x4b\x5c".as_slice(),
+            b"\x02\xfc\xaa".as_slice(),
+            b"\x01\x02\x03\x04\x05".as_slice(),
+        ] {
+            let private_key = KEYPAIR.secret.clone();
+
+            let mut binary = DcKey::to_bytes(&private_key);
+            binary.extend(garbage);
+
+            let private_key2 =
+                SignedSecretKey::from_slice(&binary).expect("Failed to ignore garbage");
+
+            assert_eq!(private_key.dc_fingerprint(), private_key2.dc_fingerprint());
+        }
+    }
+
     #[test]
     fn test_base64_roundtrip() {
         let key = KEYPAIR.public.clone();

src/lib.rs

@@ -16,7 +16,8 @@
     clippy::explicit_into_iter_loop,
     clippy::cloned_instead_of_copied
 )]
-#![cfg_attr(not(test), warn(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::indexing_slicing))]
+#![cfg_attr(not(test), forbid(clippy::string_slice))]
 #![allow(
     clippy::match_bool,
     clippy::mixed_read_write_in_expression,