add example

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.74.0
+      RUSTUP_TOOLCHAIN: 1.73.0
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -76,11 +76,11 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.74.0
+            rust: 1.73.0
           - os: windows-latest
-            rust: 1.74.0
+            rust: 1.73.0
           - os: macos-latest
-            rust: 1.74.0
+            rust: 1.73.0
 
           # Minimum Supported Rust Version = 1.70.0
           - os: ubuntu-latest
@@ -211,7 +211,7 @@ jobs:
 
       - name: Run python tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
           DCC_RS_TARGET: debug
           DCC_RS_DEV: ${{ github.workspace }}
         working-directory: python
@@ -274,6 +274,6 @@ jobs:
 
       - name: Run deltachat-rpc-client tests
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
         working-directory: deltachat-rpc-client
         run: tox -e py

.github/workflows/jsonrpc-client-npm-package.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "18"
+          node-version: "16"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/jsonrpc.yml

@@ -15,10 +15,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Use Node.js 18.x
+      - name: Use Node.js 16.x
         uses: actions/setup-node@v3
         with:
-          node-version: 18.x
+          node-version: 16.x
       - name: Add Rust cache
         uses: Swatinem/rust-cache@v2
       - name: npm install
@@ -31,7 +31,7 @@ jobs:
         working-directory: deltachat-jsonrpc/typescript
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
       - name: make sure websocket server version still builds
         working-directory: deltachat-jsonrpc
         run: cargo build --bin deltachat-jsonrpc-server --features webserver

.github/workflows/node-docs.yml

@@ -16,10 +16,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Use Node.js 18.x
+      - name: Use Node.js 16.x
         uses: actions/setup-node@v3
         with:
-          node-version: 18.x
+          node-version: 16.x
 
       - name: npm install and generate documentation
         working-directory: node

.github/workflows/node-package.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "18"
+          node-version: "16"
       - name: System info
         run: |
           rustc -vV
@@ -66,8 +66,10 @@ jobs:
     runs-on: ubuntu-latest
 
     # Build Linux prebuilds inside a container with old glibc for backwards compatibility.
-    # Debian 10 contained glibc 2.28: https://packages.debian.org/buster/libc6
-    container: debian:10
+    # Debian 10 contained glibc 2.28 at the time of the writing (2023-06-04): https://packages.debian.org/buster/libc6
+    # Ubuntu 18.04 is at the End of Standard Support since June 2023, but it contains glibc 2.27,
+    # so we are using it to support Ubuntu 18.04 setups that are still not upgraded.
+    container: ubuntu:18.04
     steps:
       # Working directory is owned by 1001:1001 by default.
       # Change it to our user.
@@ -78,7 +80,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "18"
+          node-version: "16"
       - run: apt-get update
 
       # Python is needed for node-gyp
@@ -141,7 +143,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v2
         with:
-          node-version: "18"
+          node-version: "16"
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1

.github/workflows/node-tests.yml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: "18"
+          node-version: "16"
       - name: System info
         run: |
           rustc -vV
@@ -63,5 +63,5 @@ jobs:
         working-directory: node
         run: npm run test
         env:
-          CHATMAIL_DOMAIN: ${{ secrets.CHATMAIL_DOMAIN }}
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
           NODE_OPTIONS: "--force-node-api-uncaught-exceptions-policy=true"

.github/workflows/upload-python-docs.yml

@@ -1,25 +0,0 @@
-name: Build & Deploy Documentation on py.delta.chat
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Fetch history to calculate VCS version number.
-      - name: Build Python documentation
-        run: scripts/build-python-docs.sh
-      - name: Upload to py.delta.chat
-        uses: up9cloud/action-rsync@v1.3
-        env:
-          USER: delta
-          KEY: ${{ secrets.CODESPEAK_KEY }}
-          HOST: "lists.codespeak.net"
-          SOURCE: "dist/html/"
-          TARGET: "/home/delta/build/master"

CHANGELOG.md

@@ -1,227 +1,5 @@
 # Changelog
 
-## [1.131.6] - 2023-11-21
-
-### Fixes
-
-- Fail fast if IMAP FETCH cannot be parsed instead of getting stuck in infinite loop.
-
-### Documentation
-
-- Generate deltachat-rpc-client documentation and publish it to <https://py.delta.chat>.
-
-## [1.131.5] - 2023-11-20
-
-### API-Changes
-
-- deltachat-rpc-client: Add `Message.get_sender_contact()`.
-- Turn `ContactAddress` into an owned type.
-
-### Features / Changes
-
-- Lowercase addresses in Autocrypt and Autocrypt-Gossip headers.
-- Lowercase the address in member added/removed messages.
-- Lowercase `addr` when it is set.
-- Do not replace the message with an error in square brackets when the sender is not a member of the protected group.
-
-### Fixes
-
-- `Chat::sync_contacts()`: Fetch contact addresses in a single query.
-- `Chat::rename_ex()`: Sync improved chat name to other devices.
-- Recognize `Chat-Group-Member-Added` of self case-insensitively.
-- Compare verifier addr to peerstate addr case-insensitively.
-
-### Tests
-
-- Port [Secure-Join](https://securejoin.readthedocs.io/) tests to JSON-RPC.
-
-### CI
-
-- Test with Rust 1.74.
-
-
-## [1.131.4] - 2023-11-16
-
-### Documentation
-
-- Document DC_DOWNLOAD_UNDECIPHERABLE.
-
-### Fixes
-
-- Always add "Member added" as system message.
-
-## [1.131.3] - 2023-11-15
-
-### Fixes
-
-- Update async-imap to 0.9.4 which does not ignore EOF on FETCH.
-- Reset gossiped timestamp on securejoin.
-- sync: Ignore unknown sync items to provide forward compatibility and avoid creating empty message bubbles.
-- sync: Skip sync when chat name is set to the current one.
-- Return connectivity HTML with an error when IO is stopped.
-
-## [1.131.2] - 2023-11-14
-
-### API-Changes
-
-- deltachat-rpc-client: add `Account.get_chat_by_contact()`.
-
-### Features / Changes
-
-- Do not post "... verified" messages on QR scan success.
-- Never drop better message from `apply_group_changes()`.
-
-### Fixes
-
-- Assign MDNs to the trash chat early to prevent received MDNs from creating or unblocking 1:1 chats.
-- Allow to securejoin groups when 1:1 chat with the inviter is a contact request.
-- Add "setup changed" message for verified key before the message.
-- Ignore special chats when calculating similar chats.
-
-## [1.131.1] - 2023-11-13
-
-### Fixes
-
-- Do not skip actual message parts when group change messages are inserted.
-
-## [1.131.0] - 2023-11-13
-
-### Features / Changes
-
-- Sync chat contacts across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
-- Sync creating broadcast lists across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
-- Sync Chat::name across devices ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
-- Multi-device broadcast lists ([#4953](https://github.com/deltachat/deltachat-core-rust/pull/4953)).
-
-### Fixes
-
-- Encode chat name in the `List-ID` header to avoid SMTPUTF8 errors.
-- Ignore errors from generating sync messages.
-- `Context::execute_sync_items`: Ignore all errors ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
-- Allow to send unverified securejoin messages to protected chats ([#4982](https://github.com/deltachat/deltachat-core-rust/pull/4982)).
-
-## [1.130.0] - 2023-11-10
-
-### API-Changes
-
-- Emit JoinerProgress(1000) event when Bob verifies Alice.
-- JSON-RPC: add `ContactObject.is_profile_verified` property.
-- Hide `ChatId::get_for_contact()` from public API.
-
-### Features / Changes
-
-- Add secondary verified key.
-- Add info messages about implicitly added members.
-- Treat reset state as encryption not preferred.
-- Grow sleep durations on errors in Imap::fake_idle() ([#4424](https://github.com/deltachat/deltachat-core-rust/pull/4424)).
-
-### Fixes
-
-- Mark 1:1 chat as protected when joining a group.
-- Raise lower auto-download limit to 160k.
-- Remove `Reporting-UA` from read receipts.
-- Do not apply group changes to special chats. Avoid adding members to the trash chat.
-- imap: make `UidGrouper` robust against duplicate UIDs.
-- Do not return hidden chat from `dc_get_chat_id_by_contact_id`.
-- Smtp_loop(): Don't grow timeout if interrupted early ([#4833](https://github.com/deltachat/deltachat-core-rust/pull/4833)).
-
-### Refactor
-
-- imap: Do not FETCH right after `scan_folders()`.
-- deltachat-rpc-client: Use `itertools` instead of `Lock` for thread-safe request ID generation.
-
-### Tests
-
-- Remove unused `--liveconfig` option.
-- Test chatlist can load for corrupted chats ([#4979](https://github.com/deltachat/deltachat-core-rust/pull/4979)).
-
-### Miscellaneous Tasks
-
-- Update provider-db ([#4949](https://github.com/deltachat/deltachat-core-rust/pull/4949)).
-
-## [1.129.1] - 2023-11-06
-
-### Fixes
-
-- Update tokio-imap to fix Outlook STATUS parsing bug.
-- deltachat-rpc-client: Add the Lock around request ID.
-- `apply_group_changes`: Don't implicitly delete members locally, add absent ones instead ([#4934](https://github.com/deltachat/deltachat-core-rust/pull/4934)).
-- Partial messages do not change group state ([#4900](https://github.com/deltachat/deltachat-core-rust/pull/4900)).
-
-### Tests
-
-- Group chats device synchronisation.
-
-## [1.129.0] - 2023-11-06
-
-### API-Changes
-
-- Add JSON-RPC `get_chat_id_by_contact_id` API ([#4918](https://github.com/deltachat/deltachat-core-rust/pull/4918)).
-- [**breaking**] Remove deprecated `get_verifier_addr`.
-
-### Features / Changes
-
-- Sync chat `Blocked` state, chat visibility, chat mute duration and contact blocked status across devices ([#4817](https://github.com/deltachat/deltachat-core-rust/pull/4817)).
-- Add 'group created instructions' as info message ([#4916](https://github.com/deltachat/deltachat-core-rust/pull/4916)).
-- Add hardcoded fallback DNS cache.
-
-### Fixes
-
-- Switch to `EncryptionPreference::Mutual` on a receipt of encrypted+signed message ([#4707](https://github.com/deltachat/deltachat-core-rust/pull/4707)).
-- imap: Check UIDNEXT with a STATUS command before going IDLE.
-- Allow to change verified key via "member added" message.
-- json-rpc: Return verifier even if the contact is not "verified" (Autocrypt key does not equal Secure-Join key).
-
-### Documentation
-
-- Refine `Contact::get_verifier_id` and `Contact::is_verified` documentation ([#4922](https://github.com/deltachat/deltachat-core-rust/pull/4922)).
-- Contact profile view should not use `dc_contact_is_verified()`.
-- Remove documentation for non-existing `dc_accounts_new` `os_name` param.
-
-### Refactor
-
-- Remove unused or useless code paths in Secure-Join ([#4897](https://github.com/deltachat/deltachat-core-rust/pull/4897)).
-- Improve error handling in Secure-Join code.
-- Add hostname to "no DNS resolution results" error message.
-- Accept `&str` instead of `Option<String>` in idle().
-
-## [1.128.0] - 2023-11-02
-
-### Build system
-- [**breaking**] Upgrade nodejs version to 18 ([#4903](https://github.com/deltachat/deltachat-core-rust/pull/4903)).
-
-### Features / Changes
-
-- deltachat-rpc-client: Add `Account.wait_for_incoming_msg_event()`.
-- Decrease ratelimit for .testrun.org subdomains.
-
-### Fixes
-
-- Do not fail securejoin due to unrelated pending bobstate  ([#4896](https://github.com/deltachat/deltachat-core-rust/pull/4896)).
-- Allow other verified group recipients to be unverified, only check the sender verification.
-- Remove not working attempt to recover from verified key changes.
-
-## [1.127.2] - 2023-10-29
-
-### API-Changes
-
-- [**breaking**] Jsonrpc `misc_set_draft` now requires setting the viewtype.
-- jsonrpc: Add `get_message_info_object`.
-
-### Tests
-
-- deltachat-rpc-client: Move pytest option from pyproject.toml to tox.ini and set log level.
-- deltachat-rpc-client: Test securejoin.
-- Increase pytest timeout to 10 minutes.
-- Compile deltachat-rpc-server in debug mode for tests.
-
-## [1.127.1] - 2023-10-27
-
-### API-Changes
-
-- jsonrpc: add `.is_protection_broken` to `FullChat` and `BasicChat`.
-- jsonrpc: Add `id` to `ProviderInfo`.
-
 ## [1.127.0] - 2023-10-26
 
 ### API-Changes
@@ -237,8 +15,6 @@
 - [**breaking**] Remove unused `dc_set_chat_protection()`
 - Hide `DcSecretKey` trait from the API.
 - Verified 1:1 chats ([#4315](https://github.com/deltachat/deltachat-core-rust/pull/4315)). Disabled by default, enable with `verified_one_on_one_chats` config.
-- Add api `chat::Chat::is_protection_broken`
-- Add `dc_chat_is_protection_broken()` C API.
 
 ### CI
 
@@ -1295,7 +1071,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 ### Changes
 - Look at Authentication-Results. Don't accept Autocrypt key changes
-  if they come with negative authentication results while this contact
+  if they come with negative authentiation results while this contact
   sent emails with positive authentication results in the past. #3583
 - jsonrpc in cffi also sends events now #3662
 - jsonrpc: new format for events and better typescript autocompletion
@@ -2881,7 +2657,7 @@ Bugfix release attempting to fix the [iOS build error](https://github.com/deltac
 
 - delete all consumed secure-join handshake messagess #1209 #1212
 
-- Rust-level cleanups #1218 #1217 #1210 #1205
+- rust-level cleanups #1218 #1217 #1210 #1205
 
 - python-level cleanups #1204 #1202 #1201
 
@@ -3250,16 +3026,3 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.126.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.125.0...v1.126.0
 [1.126.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.0...v1.126.1
 [1.127.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.126.1...v1.127.0
-[1.127.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.0...v1.127.1
-[1.127.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.1...v1.127.2
-[1.128.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.127.2...v1.128.0
-[1.129.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.128.0...v1.129.0
-[1.129.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.0...v1.129.1
-[1.130.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.129.1...v1.130.0
-[1.131.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.130.0...v1.131.0
-[1.131.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.0...v1.131.1
-[1.131.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.1...v1.131.2
-[1.131.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.2...v1.131.3
-[1.131.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.3...v1.131.4
-[1.131.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.4...v1.131.5
-[1.131.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.5...v1.131.6

CONTRIBUTING.md

@@ -86,11 +86,6 @@ For example:
 .with_context(|| format!("Unable to trash message {msg_id}"))
 ```
 
-All errors should be handled in one of these ways:
-- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
-- With `.log_err().ok()`.
-- Bubbled up with `?`.
-
 ### Logging
 
 For logging, use `info!`, `warn!` and `error!` macros.

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.131.6"
+version = "1.127.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.70"
@@ -32,7 +32,7 @@ format-flowed = { path = "./format-flowed" }
 ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
-async-channel = "2.0.0"
+async-channel = "1.8.0"
 async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
 async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
 async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
@@ -65,14 +65,13 @@ once_cell = "1.18.0"
 percent-encoding = "2.3"
 parking_lot = "0.12"
 pgp = { version = "0.10", default-features = false }
-pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
 quick-xml = "0.31"
 rand = "0.8"
 regex = "1.9"
 reqwest = { version = "0.11.20", features = ["json"] }
-rusqlite = { version = "0.30", features = ["sqlcipher"] }
+rusqlite = { version = "0.29", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
 serde_json = "1.0"
@@ -90,7 +89,7 @@ tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
 tokio-util = "0.7.9"
-toml = "0.8"
+toml = "0.7"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.131.6"
+version = "1.127.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -2561,7 +2561,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
- *     See https://securejoin.readthedocs.io/en/latest/new.html
+ *     See https://countermitm.readthedocs.io/en/latest/new.html
  *     for details about both protocols.
  * @return The text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
@@ -2597,7 +2597,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  *
  * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
- * See https://securejoin.readthedocs.io/en/latest/new.html
+ * See https://countermitm.readthedocs.io/en/latest/new.html
  * for details about both protocols.
  *
  * @memberof dc_context_t
@@ -2941,6 +2941,7 @@ int dc_receive_backup (dc_context_t* context, const char* qr);
  * use dc_accounts_remove_account().
  *
  * @memberof dc_accounts_t
+ * @param os_name
  * @param dir The directory to create the context-databases in.
  *     If the directory does not exist,
  *     dc_accounts_new() will try to create it.
@@ -3731,22 +3732,8 @@ int             dc_chat_can_send              (const dc_chat_t* chat);
 
 /**
  * Check if a chat is protected.
- *
- * End-to-end encryption is guaranteed in protected chats
- * and only verified contacts
- * as determined by dc_contact_is_verified()
- * can be added to protected chats.
- *
- * Protected chats are created using dc_create_group_chat()
- * by setting the 'protect' parameter to 1.
- * 1:1 chats become protected or unprotected automatically
- * if `verified_one_on_one_chats` setting is enabled.
- *
- * UI should display a green checkmark
- * in the chat title,
- * in the chatlist item
- * and in the chat profile
- * if chat protection is enabled.
+ * Protected chats contain only verified members and encryption is always enabled.
+ * Protected chats are created using dc_create_group_chat() by setting the 'protect' parameter to 1.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -4580,18 +4567,15 @@ int dc_msg_has_html (dc_msg_t* msg);
   * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
   *
   * The function returns one of:
-  * - @ref DC_DOWNLOAD_DONE           - The message does not need any further download action
-  *                                     and should be rendered as usual.
-  * - @ref DC_DOWNLOAD_AVAILABLE      - There is additional content to download.
-  *                                     In addition to the usual message rendering,
-  *                                     the UI shall show a download button that calls dc_download_full_msg()
-  * - @ref DC_DOWNLOAD_IN_PROGRESS    - Download was started with dc_download_full_msg() and is still in progress.
-  *                                     If the download fails or succeeds,
-  *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
-  *
-  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
-  *                                     It was fully downloaded, but we failed to decrypt it.
-  * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
+  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
+  *                                  and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
+  *                                  In addition to the usual message rendering,
+  *                                  the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
+  *                                  If the download fails or succeeds,
+  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
   *
   * @memberof dc_msg_t
   * @param msg The message object.
@@ -5049,16 +5033,10 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 
 
 /**
- * Check if the contact
- * can be added to verified chats,
- * i.e. has a verified key
- * and Autocrypt key matches the verified key.
+ * Check if a contact was verified. E.g. by a secure-join QR code scan
+ * and if the key has not changed since this verification.
  *
- * If contact is verified
- * UI should display green checkmark after the contact name
- * in contact list items,
- * in chat member list items
- * and in profiles if no chat with the contact exist (otherwise, use dc_chat_is_protected()).
+ * The UI may draw a checkbox or something like that beside verified contacts.
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -5068,19 +5046,32 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
 
+
 /**
- * Return the contact ID that verified a contact.
+ * Return the address that verified a contact
  *
- * If the function returns non-zero result,
- * display green checkmark in the profile and "Introduced by ..." line
- * with the name and address of the contact
- * formatted by dc_contact_get_name_n_addr.
+ * The UI may use this in addition to a checkmark showing the verification status.
+ * In case of verification chains,
+ * the last contact in the chain is shown.
+ * This is because of privacy reasons, but also as it would not help the user
+ * to see a unknown name here - where one can mostly always ask the shown name
+ * as it is directly known.
  *
- * If this function returns a verifier,
- * this does not necessarily mean
- * you can add the contact to verified chats.
- * Use dc_contact_is_verified() to check
- * if a contact can be added to a verified chat instead.
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 
+ *    A string containing the verifiers address. If it is the same address as the contact itself,
+ *    we verified the contact ourself. If it is an empty string, we don't have verifier 
+ *    information or the contact is not verified.
+ * @deprecated 2023-09-28, use dc_contact_get_verifier_id instead
+ */
+char*           dc_contact_get_verifier_addr       (dc_contact_t* contact);
+
+
+/**
+ * Return the `ContactId` that verified a contact
+ *
+ * The UI may use this in addition to a checkmark showing the verification status
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
@@ -6243,7 +6234,6 @@ void dc_event_unref(dc_event_t* event);
  * @param data2 (int) The progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
- *     1000=vg-member-added/vc-contact-confirm received
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
@@ -6436,27 +6426,22 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Download not needed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_DONE           0
+#define DC_DOWNLOAD_DONE          0
 
 /**
  * Download available, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_AVAILABLE      10
+#define DC_DOWNLOAD_AVAILABLE    10
 
 /**
  * Download failed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_FAILURE        20
-
-/**
- * Download not needed, see dc_msg_get_download_state() for details.
- */
-#define DC_DOWNLOAD_UNDECIPHERABLE 30
+#define DC_DOWNLOAD_FAILURE      20
 
 /**
  * Download in progress, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_IN_PROGRESS    1000
+#define DC_DOWNLOAD_IN_PROGRESS  1000
 
 
 
@@ -7275,11 +7260,6 @@ void dc_event_unref(dc_event_t* event);
 /// Used in info messages.
 #define DC_STR_CHAT_PROTECTION_DISABLED 171
 
-/// "Others will only see this group after you sent a first message."
-///
-/// Used as the first info messages in newly created groups.
-#define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
-
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -489,7 +489,7 @@ pub unsafe extern "C" fn dc_start_io(context: *mut dc_context_t) {
     if context.is_null() {
         return;
     }
-    let ctx = &mut *context;
+    let ctx = &*context;
 
     block_on(ctx.start_io())
 }
@@ -4119,6 +4119,23 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
         .unwrap_or_default() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_get_verifier_addr(
+    contact: *mut dc_contact_t,
+) -> *mut libc::c_char {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_get_verifier_addr()");
+        return "".strdup();
+    }
+    let ffi_contact = &*contact;
+    let ctx = &*ffi_contact.context;
+    block_on(ffi_contact.contact.get_verifier_addr(ctx))
+        .context("failed to get verifier for contact")
+        .log_err(ctx)
+        .unwrap_or_default()
+        .strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_get_verifier_id(contact: *mut dc_contact_t) -> u32 {
     if contact.is_null() {
@@ -4929,8 +4946,8 @@ pub unsafe extern "C" fn dc_accounts_start_io(accounts: *mut dc_accounts_t) {
         return;
     }
 
-    let accounts = &mut *accounts;
-    block_on(async move { accounts.write().await.start_io().await });
+    let accounts = &*accounts;
+    block_on(async move { accounts.read().await.start_io().await });
 }
 
 #[no_mangle]

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.131.6"
+version = "1.127.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -19,7 +19,7 @@ schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
 tempfile = "3.8.0"
 log = "0.4"
-async-channel = { version = "2.0.0" }
+async-channel = { version = "1.8.0" }
 futures = { version = "0.3.28" }
 serde_json = "1.0.105"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }

deltachat-jsonrpc/README.md

@@ -108,10 +108,10 @@ This will build the `deltachat-jsonrpc-server` binary and then run a test suite
 
 The test suite includes some tests that need online connectivity and a way to create test email accounts. To run these tests, talk to DeltaChat developers to get a token for the `testrun.org` service, or use a local instance of [`mailadm`](https://github.com/deltachat/docker-mailadm).
 
-Then, set the `CHATMAIL_DOMAIN` environment variable to your testing email server domain.
+Then, set the `DCC_NEW_TMP_EMAIL` environment variable to your mailadm token before running the tests.
 
 ```
-CHATMAIL_DOMAIN=chat.example.org npm run test
+DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=yourtoken npm run test
 ```
 
 #### Test Coverage

deltachat-jsonrpc/src/api/mod.rs

@@ -47,7 +47,7 @@ use types::provider_info::ProviderInfo;
 use types::reactions::JSONRPCReactions;
 use types::webxdc::WebxdcMessageInfo;
 
-use self::types::message::{MessageInfo, MessageLoadResult};
+use self::types::message::MessageLoadResult;
 use self::types::{
     chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
     location::JsonrpcLocation,
@@ -221,13 +221,13 @@ impl CommandApi {
 
     /// Starts background tasks for all accounts.
     async fn start_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.write().await.start_io().await;
+        self.accounts.read().await.start_io().await;
         Ok(())
     }
 
     /// Stops background tasks for all accounts.
     async fn stop_io_for_all_accounts(&self) -> Result<()> {
-        self.accounts.write().await.stop_io().await;
+        self.accounts.read().await.stop_io().await;
         Ok(())
     }
 
@@ -237,7 +237,7 @@ impl CommandApi {
 
     /// Starts background tasks for a single account.
     async fn start_io(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.start_io().await;
         Ok(())
     }
@@ -383,7 +383,7 @@ impl CommandApi {
     /// Configures this account with the currently set parameters.
     /// Setup the credential config before calling this.
     async fn configure(&self, account_id: u32) -> Result<()> {
-        let mut ctx = self.get_context(account_id).await?;
+        let ctx = self.get_context(account_id).await?;
         ctx.stop_io().await;
         let result = ctx.configure().await;
         if result.is_err() {
@@ -678,7 +678,7 @@ impl CommandApi {
     ///     the Verified-Group-Invite protocol is offered in the QR code;
     ///     works for protected groups as well as for normal groups.
     ///     If not set, the Setup-Contact protocol is offered in the QR code.
-    ///     See https://securejoin.readthedocs.io/en/latest/new.html
+    ///     See https://countermitm.readthedocs.io/en/latest/new.html
     ///     for details about both protocols.
     ///
     /// return format: `[code, svg]`
@@ -707,7 +707,7 @@ impl CommandApi {
     ///
     /// Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
     ///
-    /// See https://securejoin.readthedocs.io/en/latest/new.html
+    /// See https://countermitm.readthedocs.io/en/latest/new.html
     /// for details about both protocols.
     ///
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
@@ -1126,16 +1126,6 @@ impl CommandApi {
         MsgId::new(message_id).get_info(&ctx).await
     }
 
-    /// Returns additional information for single message.
-    async fn get_message_info_object(
-        &self,
-        account_id: u32,
-        message_id: u32,
-    ) -> Result<MessageInfo> {
-        let ctx = self.get_context(account_id).await?;
-        MessageInfo::from_msg_id(&ctx, MsgId::new(message_id)).await
-    }
-
     /// Returns contacts that sent read receipts and the time of reading.
     async fn get_message_read_receipts(
         &self,
@@ -1391,19 +1381,6 @@ impl CommandApi {
     //                   chat
     // ---------------------------------------------
 
-    /// Returns the [`ChatId`] for the 1:1 chat with `contact_id` if it exists.
-    ///
-    /// If it does not exist, `None` is returned.
-    async fn get_chat_id_by_contact_id(
-        &self,
-        account_id: u32,
-        contact_id: u32,
-    ) -> Result<Option<u32>> {
-        let ctx = self.get_context(account_id).await?;
-        let chat_id = ChatId::lookup_by_contact(&ctx, ContactId::new(contact_id)).await?;
-        Ok(chat_id.map(|id| id.to_u32()))
-    }
-
     /// Returns all message IDs of the given types in a chat.
     /// Typically used to show a gallery.
     ///
@@ -2056,19 +2033,13 @@ impl CommandApi {
         text: Option<String>,
         file: Option<String>,
         quoted_message_id: Option<u32>,
-        view_type: Option<MessageViewtype>,
     ) -> Result<()> {
         let ctx = self.get_context(account_id).await?;
-        let mut draft = Message::new(view_type.map_or_else(
-            || {
-                if file.is_some() {
-                    Viewtype::File
-                } else {
-                    Viewtype::Text
-                }
-            },
-            |v| v.into(),
-        ));
+        let mut draft = Message::new(if file.is_some() {
+            Viewtype::File
+        } else {
+            Viewtype::Text
+        });
         draft.set_text(text.unwrap_or_default());
         if let Some(file) = file {
             draft.set_file(file, None);

deltachat-jsonrpc/src/api/types/chat.rs

@@ -18,17 +18,6 @@ use super::contact::ContactObject;
 pub struct FullChat {
     id: u32,
     name: String,
-
-    /// True if the chat is protected.
-    ///
-    /// UI should display a green checkmark
-    /// in the chat title,
-    /// in the chat profile title and
-    /// in the chatlist item
-    /// if chat protection is enabled.
-    /// UI should also display a green checkmark
-    /// in the contact profile
-    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -42,7 +31,6 @@ pub struct FullChat {
     fresh_message_counter: usize,
     // is_group - please check over chat.type in frontend instead
     is_contact_request: bool,
-    is_protection_broken: bool,
     is_device_chat: bool,
     self_in_group: bool,
     is_muted: bool,
@@ -112,7 +100,6 @@ impl FullChat {
             color,
             fresh_message_counter,
             is_contact_request: chat.is_contact_request(),
-            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             self_in_group: contact_ids.contains(&ContactId::SELF),
             is_muted: chat.is_muted(),
@@ -139,17 +126,6 @@ impl FullChat {
 pub struct BasicChat {
     id: u32,
     name: String,
-
-    /// True if the chat is protected.
-    ///
-    /// UI should display a green checkmark
-    /// in the chat title,
-    /// in the chat profile title and
-    /// in the chatlist item
-    /// if chat protection is enabled.
-    /// UI should also display a green checkmark
-    /// in the contact profile
-    /// if 1:1 chat with this contact exists and is protected.
     is_protected: bool,
     profile_image: Option<String>, //BLOBS ?
     archived: bool,
@@ -158,7 +134,6 @@ pub struct BasicChat {
     is_self_talk: bool,
     color: String,
     is_contact_request: bool,
-    is_protection_broken: bool,
     is_device_chat: bool,
     is_muted: bool,
 }
@@ -185,7 +160,6 @@ impl BasicChat {
             is_self_talk: chat.is_self_talk(),
             color,
             is_contact_request: chat.is_contact_request(),
-            is_protection_broken: chat.is_protection_broken(),
             is_device_chat: chat.is_device_talk(),
             is_muted: chat.is_muted(),
         })

deltachat-jsonrpc/src/api/types/contact.rs

@@ -19,30 +19,11 @@ pub struct ContactObject {
     profile_image: Option<String>, // BLOBS
     name_and_addr: String,
     is_blocked: bool,
-
-    /// True if the contact can be added to verified groups.
-    ///
-    /// If this is true
-    /// UI should display green checkmark after the contact name
-    /// in contact list items,
-    /// in chat member list items
-    /// and in profiles if no chat with the contact exist.
     is_verified: bool,
-
-    /// True if the contact profile title should have a green checkmark.
-    ///
-    /// This indicates whether 1:1 chat has a green checkmark
-    /// or will have a green checkmark if created.
-    is_profile_verified: bool,
-
-    /// The ID of the contact that verified this contact.
-    ///
-    /// If this is present,
-    /// display a green checkmark and "Introduced by ..."
-    /// string followed by the verifier contact name and address
-    /// in the contact profile.
+    /// the address that verified this contact
+    verifier_addr: Option<String>,
+    /// the id of the contact that verified this contact
     verifier_id: Option<u32>,
-
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
@@ -58,12 +39,18 @@ impl ContactObject {
             None => None,
         };
         let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
-        let is_profile_verified = contact.is_profile_verified(context).await?;
 
-        let verifier_id = contact
-            .get_verifier_id(context)
-            .await?
-            .map(|contact_id| contact_id.to_u32());
+        let (verifier_addr, verifier_id) = if is_verified {
+            (
+                contact.get_verifier_addr(context).await?,
+                contact
+                    .get_verifier_id(context)
+                    .await?
+                    .map(|contact_id| contact_id.to_u32()),
+            )
+        } else {
+            (None, None)
+        };
 
         Ok(ContactObject {
             address: contact.get_addr().to_owned(),
@@ -77,7 +64,7 @@ impl ContactObject {
             name_and_addr: contact.get_name_n_addr(),
             is_blocked: contact.is_blocked(),
             is_verified,
-            is_profile_verified,
+            verifier_addr,
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),

deltachat-jsonrpc/src/api/types/message.rs

@@ -34,10 +34,6 @@ pub struct MessageObject {
     quote: Option<MessageQuote>,
     parent_id: Option<u32>,
 
-    /// Contents of `Message-ID` header.
-    /// Possibly a fake unique string if incoming message did not have a `Message-ID` header.
-    message_id: String,
-
     text: String,
     has_location: bool,
     has_html: bool,
@@ -178,8 +174,6 @@ impl MessageObject {
             Some(reactions.into())
         };
 
-        let message_id = message.rfc724_mid().to_string();
-
         Ok(MessageObject {
             id: msg_id.to_u32(),
             chat_id: message.get_chat_id().to_u32(),
@@ -239,8 +233,6 @@ impl MessageObject {
             download_state,
 
             reactions,
-
-            message_id,
         })
     }
 }
@@ -560,71 +552,3 @@ pub struct MessageReadReceipt {
     pub contact_id: u32,
     pub timestamp: i64,
 }
-
-#[derive(Serialize, TypeDef, schemars::JsonSchema)]
-#[serde(rename_all = "camelCase")]
-pub struct MessageInfo {
-    rawtext: String,
-    ephemeral_timer: EphemeralTimer,
-    /// When message is ephemeral this contains the timestamp of the message expiry
-    ephemeral_timestamp: Option<i64>,
-    error: Option<String>,
-    rfc724_mid: String,
-    server_urls: Vec<String>,
-    hop_info: Option<String>,
-}
-
-impl MessageInfo {
-    pub async fn from_msg_id(context: &Context, msg_id: MsgId) -> Result<Self> {
-        let message = Message::load_from_db(context, msg_id).await?;
-        let rawtext = msg_id.rawtext(context).await?;
-        let ephemeral_timer = message.get_ephemeral_timer().into();
-        let ephemeral_timestamp = match message.get_ephemeral_timer() {
-            deltachat::ephemeral::Timer::Disabled => None,
-            deltachat::ephemeral::Timer::Enabled { .. } => Some(message.get_ephemeral_timestamp()),
-        };
-
-        let server_urls =
-            MsgId::get_info_server_urls(context, message.rfc724_mid().to_owned()).await?;
-
-        let hop_info = msg_id.hop_info(context).await?;
-
-        Ok(Self {
-            rawtext,
-            ephemeral_timer,
-            ephemeral_timestamp,
-            error: message.error(),
-            rfc724_mid: message.rfc724_mid().to_owned(),
-            server_urls,
-            hop_info,
-        })
-    }
-}
-
-#[derive(
-    Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize, TypeDef, schemars::JsonSchema,
-)]
-#[serde(rename_all = "camelCase", tag = "variant")]
-pub enum EphemeralTimer {
-    /// Timer is disabled.
-    Disabled,
-
-    /// Timer is enabled.
-    Enabled {
-        /// Timer duration in seconds.
-        ///
-        /// The value cannot be 0.
-        duration: u32,
-    },
-}
-
-impl From<deltachat::ephemeral::Timer> for EphemeralTimer {
-    fn from(value: deltachat::ephemeral::Timer) -> Self {
-        match value {
-            deltachat::ephemeral::Timer::Disabled => EphemeralTimer::Disabled,
-            deltachat::ephemeral::Timer::Enabled { duration } => {
-                EphemeralTimer::Enabled { duration }
-            }
-        }
-    }
-}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -6,8 +6,6 @@ use typescript_type_def::TypeDef;
 #[derive(Serialize, TypeDef, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
 pub struct ProviderInfo {
-    /// Unique ID, corresponding to provider database filename.
-    pub id: String,
     pub before_login_hint: String,
     pub overview_page: String,
     pub status: u32, // in reality this is an enum, but for simplicity and because it gets converted into a number anyway, we use an u32 here.
@@ -16,7 +14,6 @@ pub struct ProviderInfo {
 impl ProviderInfo {
     pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
         provider.map(|p| ProviderInfo {
-            id: p.id.to_owned(),
             before_login_hint: p.before_login_hint.to_owned(),
             overview_page: p.overview_page.to_owned(),
             status: p.status.to_u32().unwrap(),

deltachat-jsonrpc/src/lib.rs

@@ -17,7 +17,7 @@ mod tests {
         let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, receiver) = unbounded::<String>();
+        let (sender, mut receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -36,17 +36,17 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.recv().await?;
+            let result = receiver.next().await;
             println!("{result:?}");
-            assert_eq!(result, response.to_owned());
+            assert_eq!(result, Some(response.to_owned()));
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
             session.handle_incoming(request).await;
-            let result = receiver.recv().await?;
+            let result = receiver.next().await;
             println!("{result:?}");
-            assert_eq!(result, response.to_owned());
+            assert_eq!(result, Some(response.to_owned()));
         }
 
         Ok(())
@@ -59,7 +59,7 @@ mod tests {
         let accounts = Accounts::new(tmp_dir, writable).await?;
         let api = CommandApi::new(accounts);
 
-        let (sender, receiver) = unbounded::<String>();
+        let (sender, mut receiver) = unbounded::<String>();
 
         let (client, mut rx) = RpcClient::new();
         let session = RpcSession::new(client, api);
@@ -78,15 +78,15 @@ mod tests {
             let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
             let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
             session.handle_incoming(request).await;
-            let result = receiver.recv().await?;
-            assert_eq!(result, response.to_owned());
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
         }
         {
             let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
             let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
             session.handle_incoming(request).await;
-            let result = receiver.recv().await?;
-            assert_eq!(result, response.to_owned());
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
         }
 
         Ok(())

deltachat-jsonrpc/src/webserver.rs

@@ -28,7 +28,7 @@ async fn main() -> Result<(), std::io::Error> {
         .layer(Extension(state.clone()));
 
     tokio::spawn(async move {
-        state.accounts.write().await.start_io().await;
+        state.accounts.read().await.start_io().await;
     });
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));

deltachat-jsonrpc/typescript/package.json

@@ -9,6 +9,7 @@
     "@types/chai": "^4.2.21",
     "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^9.0.0",
+    "@types/node-fetch": "^2.5.7",
     "@types/ws": "^7.2.4",
     "c8": "^7.10.0",
     "chai": "^4.3.4",
@@ -16,6 +17,7 @@
     "esbuild": "^0.17.9",
     "http-server": "^14.1.1",
     "mocha": "^9.1.1",
+    "node-fetch": "^2.6.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^2.6.2",
     "typedoc": "^0.23.2",
@@ -53,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.131.6"
+  "version": "1.127.0"
 }

deltachat-jsonrpc/typescript/test/basic.ts

@@ -79,9 +79,6 @@ describe("basic tests", () => {
       accountId = await dc.rpc.addAccount();
     });
     it("should block and unblock contact", async function () {
-      // Cannot send sync messages to self as we do not have a self address.
-      await dc.rpc.setConfig(accountId, "sync_msgs", "0");
-
       const contactId = await dc.rpc.createContact(
         accountId,
         "example@delta.chat",

deltachat-jsonrpc/typescript/test/online.ts

@@ -13,16 +13,16 @@ describe("online tests", function () {
 
   before(async function () {
     this.timeout(60000);
-    if (!process.env.CHATMAIL_DOMAIN) {
+    if (!process.env.DCC_NEW_TMP_EMAIL) {
       if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
         console.error(
-          "CAN NOT RUN COVERAGE correctly: Missing CHATMAIL_DOMAIN environment variable!\n\n",
+          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
           "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
         );
         process.exit(1);
       }
       console.log(
-        "Missing CHATMAIL_DOMAIN environment variable!, skip integration tests"
+        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests"
       );
       this.skip();
     }
@@ -33,7 +33,7 @@ describe("online tests", function () {
       if (kind !== "Info") console.log(contextId, kind);
     });
 
-    account1 = createTempUser(process.env.CHATMAIL_DOMAIN);
+    account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
     if (!account1 || !account1.email || !account1.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"
@@ -41,7 +41,7 @@ describe("online tests", function () {
       this.skip();
     }
 
-    account2 = createTempUser(process.env.CHATMAIL_DOMAIN);
+    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
     if (!account2 || !account2.email || !account2.password) {
       console.log(
         "We didn't got back an account2 from the api, skip integration tests"

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -2,6 +2,7 @@ import { tmpdir } from "os";
 import { join, resolve } from "path";
 import { mkdtemp, rm } from "fs/promises";
 import { spawn, exec } from "child_process";
+import fetch from "node-fetch";
 import { Readable, Writable } from "node:stream";
 
 export type RpcServerHandle = {
@@ -56,14 +57,15 @@ export async function startServer(): Promise<RpcServerHandle> {
   };
 }
 
-export function createTempUser(chatmailDomain: String) {
-  const charset = "2345789acdefghjkmnpqrstuvwxyz";
-  let user = "ci-";
-  for (let i = 0; i < 6; i++) {
-    user += charset[Math.floor(Math.random() * charset.length)];
-  }
-  const email = user + "@" + chatmailDomain;
-  return { email: email, password: user + "$" + user };
+export async function createTempUser(url: string) {
+  const response = await fetch(url, {
+    method: "POST",
+    headers: {
+      "cache-control": "no-cache",
+    },
+  });
+  if (!response.ok) throw new Error("Received invalid response");
+  return response.json();
 }
 
 function getTargetDir(): Promise<string> {

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.131.6"
+version = "1.127.0"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -11,7 +11,7 @@ deltachat = { path = "..", features = ["internals"]}
 dirs = "5"
 log = "0.4.20"
 pretty_env_logger = "0.5"
-rusqlite = "0.30"
+rusqlite = "0.29"
 rustyline = "12"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 

deltachat-repl/src/main.rs

@@ -401,7 +401,7 @@ enum ExitResult {
 
 async fn handle_cmd(
     line: &str,
-    mut ctx: Context,
+    ctx: Context,
     selected_chat: &mut ChatId,
 ) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');

deltachat-rpc-client/README.md

@@ -25,7 +25,7 @@ $ pip install .
 ## Testing
 
 1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
-2. Run `CHATMAIL_DOMAIN=nine.testrun.org PATH="../target/debug:$PATH" tox`.
+2. Run `PATH="../target/debug:$PATH" tox`.
 
 Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
 
@@ -48,3 +48,7 @@ $ python
 'awesome'
 >>> rpc.close()
 ```
+
+## Usage from async code
+
+See the [echobot_async.py](./examples/echobot_async.py) example.

deltachat-rpc-client/examples/echobot_async.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+"""
+Example asynchronous echo bot
+"""
+import asyncio
+import logging
+import sys
+
+from deltachat_rpc_client import EventType, Rpc, SpecialContactId
+
+
+async def main():
+    async with Rpc() as rpc:
+        system_info = await rpc.get_system_info()
+        logging.info("Running deltachat core %s", system_info["deltachat_core_version"])
+
+        account_ids = await rpc.get_all_account_ids()
+        accid = account_ids[0] if account_ids else await rpc.add_account()
+
+        await rpc.set_config(accid, "bot", "1")
+        if not await rpc.is_configured(accid):
+            logging.info("Account is not configured, configuring")
+            await rpc.set_config(accid, "addr", sys.argv[1])
+            await rpc.set_config(accid, "mail_pw", sys.argv[2])
+            await rpc.configure(accid)
+            logging.info("Configured")
+        else:
+            logging.info("Account is already configured")
+            await rpc.start_io(accid)
+
+        async def process_messages():
+            for msgid in await rpc.get_next_msgs(accid):
+                msg = await rpc.get_message(accid, msgid)
+                if msg["from_id"] != SpecialContactId.SELF and not msg["is_bot"] and not msg["is_info"]:
+                    await rpc.misc_send_text_message(accid, msg["chat_id"], msg["text"])
+                await rpc.markseen_msgs(accid, [msgid])
+
+        # Process old messages.
+        await process_messages()
+
+        while True:
+            event = await rpc.wait_for_event(accid)
+            if event["kind"] == EventType.INFO:
+                logging.info("%s", event["msg"])
+            elif event["kind"] == EventType.WARNING:
+                logging.warning("%s", event["msg"])
+            elif event["kind"] == EventType.ERROR:
+                logging.error("%s", event["msg"])
+            elif event["kind"] == EventType.INCOMING_MSG:
+                logging.info("Got an incoming message (id=%s)", event["msg_id"])
+                await process_messages()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    asyncio.run(main())

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -40,13 +40,13 @@ def main():
 
         while True:
             event = account.wait_for_event()
-            if event["kind"] == EventType.INFO:
+            if event["type"] == EventType.INFO:
                 logging.info("%s", event["msg"])
-            elif event["kind"] == EventType.WARNING:
+            elif event["type"] == EventType.WARNING:
                 logging.warning("%s", event["msg"])
-            elif event["kind"] == EventType.ERROR:
+            elif event["type"] == EventType.ERROR:
                 logging.error("%s", event["msg"])
-            elif event["kind"] == EventType.INCOMING_MSG:
+            elif event["type"] == EventType.INCOMING_MSG:
                 logging.info("Got an incoming message")
                 process_messages()
 

deltachat-rpc-client/pyproject.toml

@@ -71,3 +71,6 @@ line-length = 120
 
 [tool.isort]
 profile = "black"
+
+[tool.pytest.ini_options]
+log_cli = true

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -4,7 +4,7 @@ from warnings import warn
 
 from ._utils import AttrDict
 from .chat import Chat
-from .const import ChatlistFlag, ContactFlag, EventType, SpecialContactId
+from .const import ChatlistFlag, ContactFlag, SpecialContactId
 from .contact import Contact
 from .message import Message
 
@@ -111,20 +111,6 @@ class Account:
         contacts = self._rpc.get_blocked_contacts(self.id)
         return [AttrDict(contact=Contact(self, contact["id"]), **contact) for contact in contacts]
 
-    def get_chat_by_contact(self, contact: Union[int, Contact]) -> Optional[Chat]:
-        """Return 1:1 chat for a contact if it exists."""
-        if isinstance(contact, Contact):
-            assert contact.account == self
-            contact_id = contact.id
-        elif isinstance(contact, int):
-            contact_id = contact
-        else:
-            raise ValueError(f"{contact!r} is not a contact")
-        chat_id = self._rpc.get_chat_id_by_contact_id(self.id, contact_id)
-        if chat_id:
-            return Chat(self, chat_id)
-        return None
-
     def get_contacts(
         self,
         query: Optional[str] = None,
@@ -218,7 +204,7 @@ class Account:
         The function returns immediately and the handshake runs in background, sending
         and receiving several messages.
         Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
-        See https://securejoin.readthedocs.io/en/latest/new.html for protocol details.
+        See https://countermitm.readthedocs.io/en/latest/new.html for protocol details.
 
         :param qrdata: The text of the scanned QR code.
         """
@@ -264,25 +250,6 @@ class Account:
         next_msg_ids = self._rpc.wait_next_msgs(self.id)
         return [Message(self, msg_id) for msg_id in next_msg_ids]
 
-    def wait_for_incoming_msg_event(self):
-        """Wait for incoming message event and return it."""
-        while True:
-            event = self.wait_for_event()
-            if event.kind == EventType.INCOMING_MSG:
-                return event
-
-    def wait_for_securejoin_inviter_success(self):
-        while True:
-            event = self.wait_for_event()
-            if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
-                break
-
-    def wait_for_securejoin_joiner_success(self):
-        while True:
-            event = self.wait_for_event()
-            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
-                break
-
     def get_fresh_messages_in_arrival_order(self) -> List[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -160,12 +160,11 @@ class Chat:
         text: Optional[str] = None,
         file: Optional[str] = None,
         quoted_msg: Optional[int] = None,
-        viewtype: Optional[str] = None,
     ) -> None:
         """Set draft message."""
         if isinstance(quoted_msg, Message):
             quoted_msg = quoted_msg.id
-        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg, viewtype)
+        self._rpc.misc_set_draft(self.account.id, self.id, text, file, quoted_msg)
 
     def remove_draft(self) -> None:
         """Remove draft message."""

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -195,7 +195,7 @@ class Client:
 
 
 class Bot(Client):
-    """Simple bot implementation that listens to events of a single account."""
+    """Simple bot implementation that listent to events of a single account."""
 
     def configure(self, email: str, password: str, **kwargs) -> None:
         kwargs.setdefault("bot", "1")

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -42,10 +42,6 @@ class Message:
             return AttrDict(reactions)
         return None
 
-    def get_sender_contact(self) -> Contact:
-        from_id = self.get_snapshot().from_id
-        return self.account.get_contact_by_id(from_id)
-
     def mark_seen(self) -> None:
         """Mark the message as seen."""
         self._rpc.markseen_msgs(self.account.id, [self.id])

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -1,5 +1,6 @@
+import json
 import os
-import random
+import urllib.request
 from typing import AsyncGenerator, List, Optional
 
 import pytest
@@ -9,11 +10,12 @@ from .rpc import Rpc
 
 
 def get_temp_credentials() -> dict:
-    domain = os.getenv("CHATMAIL_DOMAIN")
-    username = "ci-" + "".join(random.choice("2345789acdefghjkmnpqrstuvwxyz") for i in range(6))
-    password = f"{username}${username}"
-    addr = f"{username}@{domain}"
-    return {"email": addr, "password": password}
+    url = os.getenv("DCC_NEW_TMP_EMAIL")
+    assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
+
+    request = urllib.request.Request(url, method="POST")
+    with urllib.request.urlopen(request, timeout=60) as f:
+        return json.load(f)
 
 
 class ACFactory:
@@ -21,9 +23,7 @@ class ACFactory:
         self.deltachat = deltachat
 
     def get_unconfigured_account(self) -> Account:
-        account = self.deltachat.add_account()
-        account.set_config("verified_one_on_one_chats", "1")
-        return account
+        return self.deltachat.add_account()
 
     def get_unconfigured_bot(self) -> Bot:
         return Bot(self.get_unconfigured_account())
@@ -61,16 +61,6 @@ class ACFactory:
     def get_online_accounts(self, num: int) -> List[Account]:
         return [self.get_online_account() for _ in range(num)]
 
-    def resetup_account(self, ac: Account) -> Account:
-        """Resetup account from scratch, losing the encryption key."""
-        ac.stop_io()
-        ac_clone = self.get_unconfigured_account()
-        for i in ["addr", "mail_pw"]:
-            ac_clone.set_config(i, ac.get_config(i))
-        ac.remove()
-        ac_clone.configure()
-        return ac_clone
-
     def send_message(
         self,
         to_account: Account,

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,4 +1,4 @@
-import itertools
+import asyncio
 import json
 import logging
 import os
@@ -6,7 +6,7 @@ import subprocess
 import sys
 from queue import Queue
 from threading import Event, Thread
-from typing import Any, Dict, Iterator, Optional
+from typing import Any, Dict, Optional
 
 
 class JsonRpcError(Exception):
@@ -24,7 +24,7 @@ class Rpc:
 
         self._kwargs = kwargs
         self.process: subprocess.Popen
-        self.id_iterator: Iterator[int]
+        self.id: int
         self.event_queues: Dict[int, Queue]
         # Map from request ID to `threading.Event`.
         self.request_events: Dict[int, Event]
@@ -36,6 +36,10 @@ class Rpc:
         self.writer_thread: Thread
         self.events_thread: Thread
 
+    def get_async_rpc(self) -> "AsyncRpc":
+        """Get asynchronous wrapper to use the RPC methods from async code."""
+        return AsyncRpc(self)
+
     def start(self) -> None:
         if sys.version_info >= (3, 11):
             self.process = subprocess.Popen(
@@ -55,7 +59,7 @@ class Rpc:
                 preexec_fn=os.setpgrp,  # noqa: PLW1509
                 **self._kwargs,
             )
-        self.id_iterator = itertools.count(start=1)
+        self.id = 0
         self.event_queues = {}
         self.request_events = {}
         self.request_results = {}
@@ -85,6 +89,13 @@ class Rpc:
     def __exit__(self, _exc_type, _exc, _tb):
         self.close()
 
+    async def __aenter__(self):
+        self.__enter__()
+        return self.get_async_rpc()
+
+    async def __aexit__(self, _exc_type, _exc, _tb):
+        self.__exit__(_exc_type, _exc, _tb)
+
     def reader_loop(self) -> None:
         try:
             while True:
@@ -132,9 +143,7 @@ class Rpc:
                 event = self.get_next_event()
                 account_id = event["contextId"]
                 queue = self.get_queue(account_id)
-                event = event["event"]
-                logging.debug("account_id=%d got an event %s", account_id, event)
-                queue.put(event)
+                queue.put(event["event"])
         except Exception:
             # Log an exception if the event loop dies.
             logging.exception("Exception in the event loop")
@@ -146,12 +155,14 @@ class Rpc:
 
     def __getattr__(self, attr: str):
         def method(*args) -> Any:
-            request_id = next(self.id_iterator)
+            self.id += 1
+            request_id = self.id
+
             request = {
                 "jsonrpc": "2.0",
                 "method": attr,
                 "params": args,
-                "id": request_id,
+                "id": self.id,
             }
             event = Event()
             self.request_events[request_id] = event
@@ -166,3 +177,19 @@ class Rpc:
             return None
 
         return method
+
+
+class AsyncRpc:
+    def __init__(self, sync_rpc: Rpc) -> None:
+        self._sync_rpc = sync_rpc
+
+    def __getattr__(self, attr: str) -> Any:
+        sync_method = getattr(self._sync_rpc, attr)
+        if sync_method:
+
+            async def method(*args) -> Any:
+                loop = asyncio.get_event_loop()
+                return await loop.run_in_executor(None, sync_method, *args)
+
+            return method
+        return None

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,431 +0,0 @@
-import logging
-
-from deltachat_rpc_client import Chat, SpecialContactId
-
-
-def test_qr_setup_contact(acfactory) -> None:
-    alice, bob = acfactory.get_online_accounts(2)
-
-    qr_code, _svg = alice.get_qr_code()
-    bob.secure_join(qr_code)
-
-    alice.wait_for_securejoin_inviter_success()
-
-    # Test that Alice verified Bob's profile.
-    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
-    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
-    assert alice_contact_bob_snapshot.is_verified
-
-    bob.wait_for_securejoin_joiner_success()
-
-    # Test that Bob verified Alice's profile.
-    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
-    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
-    assert bob_contact_alice_snapshot.is_verified
-
-
-def test_qr_securejoin(acfactory):
-    alice, bob = acfactory.get_online_accounts(2)
-
-    logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=True)
-    assert alice_chat.get_basic_snapshot().is_protected
-
-    logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
-    bob.secure_join(qr_code)
-
-    # Check that at least some of the handshake messages are deleted.
-    for ac in [alice, bob]:
-        while True:
-            event = ac.wait_for_event()
-            if event["kind"] == "ImapMessageDeleted":
-                break
-
-    alice.wait_for_securejoin_inviter_success()
-
-    # Test that Alice verified Bob's profile.
-    alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
-    alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
-    assert alice_contact_bob_snapshot.is_verified
-
-    bob.wait_for_securejoin_joiner_success()
-
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
-    assert snapshot.chat.get_basic_snapshot().is_protected
-
-    # Test that Bob verified Alice's profile.
-    bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
-    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
-    assert bob_contact_alice_snapshot.is_verified
-
-
-def test_qr_securejoin_contact_request(acfactory) -> None:
-    """Alice invites Bob to a group when Bob's chat with Alice is in a contact request mode."""
-    alice, bob = acfactory.get_online_accounts(2)
-
-    bob_addr = bob.get_config("addr")
-    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
-    alice_chat_bob = alice_contact_bob.create_chat()
-    alice_chat_bob.send_text("Hello!")
-
-    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hello!"
-    bob_chat_alice = snapshot.chat
-    assert bob_chat_alice.get_basic_snapshot().is_contact_request
-
-    alice_chat = alice.create_group("Verified group", protect=True)
-    logging.info("Bob joins verified group")
-    qr_code, _svg = alice_chat.get_qr_code()
-    bob.secure_join(qr_code)
-    while True:
-        event = bob.wait_for_event()
-        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
-            break
-
-    # Chat stays being a contact request.
-    assert bob_chat_alice.get_basic_snapshot().is_contact_request
-
-
-def test_qr_readreceipt(acfactory) -> None:
-    alice, bob, charlie = acfactory.get_online_accounts(3)
-
-    logging.info("Bob and Charlie setup contact with Alice")
-    qr_code, _svg = alice.get_qr_code()
-
-    bob.secure_join(qr_code)
-    charlie.secure_join(qr_code)
-
-    for joiner in [bob, charlie]:
-        joiner.wait_for_securejoin_joiner_success()
-
-    logging.info("Alice creates a verified group")
-    group = alice.create_group("Group", protect=True)
-
-    bob_addr = bob.get_config("addr")
-    charlie_addr = charlie.get_config("addr")
-
-    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
-    alice_contact_charlie = alice.create_contact(charlie_addr, "Charlie")
-
-    group.add_contact(alice_contact_bob)
-    group.add_contact(alice_contact_charlie)
-
-    # Promote a group.
-    group.send_message(text="Hello")
-
-    logging.info("Bob and Charlie receive a group")
-
-    bob_msg_id = bob.wait_for_incoming_msg_event().msg_id
-    bob_message = bob.get_message_by_id(bob_msg_id)
-    bob_snapshot = bob_message.get_snapshot()
-    assert bob_snapshot.text == "Hello"
-
-    # Charlie receives the same "Hello" message as Bob.
-    charlie.wait_for_incoming_msg_event()
-
-    logging.info("Bob sends a message to the group")
-
-    bob_out_message = bob_snapshot.chat.send_message(text="Hi from Bob!")
-
-    charlie_msg_id = charlie.wait_for_incoming_msg_event().msg_id
-    charlie_message = charlie.get_message_by_id(charlie_msg_id)
-    charlie_snapshot = charlie_message.get_snapshot()
-    assert charlie_snapshot.text == "Hi from Bob!"
-
-    bob_contact_charlie = bob.create_contact(charlie_addr, "Charlie")
-    assert not bob.get_chat_by_contact(bob_contact_charlie)
-
-    logging.info("Charlie reads Bob's message")
-    charlie_message.mark_seen()
-
-    while True:
-        event = bob.wait_for_event()
-        if event["kind"] == "MsgRead" and event["msg_id"] == bob_out_message.id:
-            break
-
-    # Receiving a read receipt from Charlie
-    # should not unblock hidden chat with Charlie for Bob.
-    assert not bob.get_chat_by_contact(bob_contact_charlie)
-
-
-def test_verified_group_recovery(acfactory) -> None:
-    """Tests verified group recovery by reverifying a member and sending a message in a group."""
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    logging.info("ac1 creates verified group")
-    chat = ac1.create_group("Verified group", protect=True)
-    assert chat.get_basic_snapshot().is_protected
-
-    logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
-    ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    # ac1 has ac2 directly verified.
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
-
-    logging.info("ac3 joins verified group")
-    ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    logging.info("ac2 logs in on a new device")
-    ac2 = acfactory.resetup_account(ac2)
-
-    logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
-    ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
-
-    logging.info("ac3 sends a message to the group")
-    assert len(ac3_chat.get_contacts()) == 3
-    ac3_chat.send_text("Hi!")
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Hi!"
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.text == "Hi!"
-
-    # ac1 contact is verified for ac2 because ac3 gossiped ac1 key in the "Hi!" message.
-    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
-    assert ac1_contact.get_snapshot().is_verified
-
-    # ac2 can write messages to the group.
-    snapshot.chat.send_text("Works again!")
-
-    snapshot = ac3.get_message_by_id(ac3.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    ac1_chat_messages = snapshot.chat.get_messages()
-    ac2_addr = ac2.get_config("addr")
-    assert ac1_chat_messages[-2].get_snapshot().text == f"Changed setup for {ac2_addr}"
-
-    # ac2 is now verified by ac3 for ac1
-    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
-
-
-def test_verified_group_member_added_recovery(acfactory) -> None:
-    """Tests verified group recovery by reverifiying than removing and adding a member back."""
-    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
-
-    logging.info("ac1 creates verified group")
-    chat = ac1.create_group("Verified group", protect=True)
-    assert chat.get_basic_snapshot().is_protected
-
-    logging.info("ac2 joins verified group")
-    qr_code, _svg = chat.get_qr_code()
-    ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    # ac1 has ac2 directly verified.
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == SpecialContactId.SELF
-
-    logging.info("ac3 joins verified group")
-    ac3_chat = ac3.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    logging.info("ac2 logs in on a new device")
-    ac2 = acfactory.resetup_account(ac2)
-
-    logging.info("ac2 reverifies with ac3")
-    qr_code, _svg = ac3.get_qr_code()
-    ac2.secure_join(qr_code)
-
-    ac3.wait_for_securejoin_inviter_success()
-
-    logging.info("ac3 sends a message to the group")
-    assert len(ac3_chat.get_contacts()) == 3
-    ac3_chat.send_text("Hi!")
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("Received message %s", snapshot.text)
-    assert snapshot.text == "Hi!"
-
-    ac1.wait_for_incoming_msg_event()  # Hi!
-
-    ac3_contact_ac2 = ac3.get_contact_by_addr(ac2.get_config("addr"))
-    ac3_chat.remove_contact(ac3_contact_ac2)
-    ac3_chat.add_contact(ac3_contact_ac2)
-
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert "removed" in snapshot.text
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert "removed" in snapshot.text
-
-    event = ac2.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    chat_id = event.chat_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "added" in snapshot.text
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert "added" in snapshot.text
-
-    chat = Chat(ac2, chat_id)
-    chat.send_text("Works again!")
-
-    msg_id = ac3.wait_for_incoming_msg_event().msg_id
-    message = ac3.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert snapshot.text == "Works again!"
-
-    ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
-    ac1_contact_ac2_snapshot = ac1_contact_ac2.get_snapshot()
-    assert ac1_contact_ac2_snapshot.is_verified
-    assert ac1_contact_ac2_snapshot.verifier_id == ac1.get_contact_by_addr(ac3.get_config("addr")).id
-
-    # ac2 is now verified by ac3 for ac1
-    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
-
-
-def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
-    """Regression test for
-    issue <https://github.com/deltachat/deltachat-core-rust/issues/4894>.
-    """
-    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
-
-    logging.info("ac3: verify with ac2")
-    qr_code, _svg = ac2.get_qr_code()
-    ac3.secure_join(qr_code)
-    ac2.wait_for_securejoin_inviter_success()
-
-    # in order for ac2 to have pending bobstate with a verified group
-    # we first create a fully joined verified group, and then start
-    # joining a second time but interrupt it, to create pending bob state
-
-    logging.info("ac1: create verified group that ac2 fully joins")
-    ch1 = ac1.create_group("Group", protect=True)
-    qr_code, _svg = ch1.get_qr_code()
-    ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    # ensure ac1 can write and ac2 receives messages in verified chat
-    ch1.send_text("ac1 says hello")
-    while 1:
-        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-        if snapshot.text == "ac1 says hello":
-            assert snapshot.chat.get_basic_snapshot().is_protected
-            break
-
-    logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    qr_code, _svg = ch1.get_qr_code()
-    ac2.secure_join(qr_code)
-    ac1.remove()
-    logging.info("ac2 now has pending bobstate but ac1 is shutoff")
-
-    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
-    assert ac3.get_contact_by_addr(ac2.get_config("addr")).get_snapshot().is_verified
-    assert ac2.get_contact_by_addr(ac3.get_config("addr")).get_snapshot().is_verified
-
-    logging.info("ac3: create a verified group VG with ac2")
-    vg = ac3.create_group("ac3-created", protect=True)
-    vg.add_contact(ac3.get_contact_by_addr(ac2.get_config("addr")))
-
-    # ensure ac2 receives message in VG
-    vg.send_text("hello")
-    while 1:
-        msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-        if msg.text == "hello":
-            assert msg.chat.get_basic_snapshot().is_protected
-            break
-
-    logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    qr_code, _svg = vg.get_qr_code()
-    ac4.secure_join(qr_code)
-    ac3.wait_for_securejoin_inviter_success()
-    while 1:
-        ev = ac2.wait_for_event()
-        if "added by unrelated SecureJoin" in str(ev):
-            return
-
-
-def test_qr_new_group_unblocked(acfactory):
-    """Regression test for a bug introduced in core v1.113.0.
-    ac2 scans a verified group QR code created by ac1.
-    This results in creation of a blocked 1:1 chat with ac1 on ac2,
-    but ac1 contact is not blocked on ac2.
-    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
-    ac2 should receive a message and create a contact request for the group.
-    Due to a bug previously ac2 created a blocked group.
-    """
-
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1_chat = ac1.create_group("Group for joining", protect=True)
-    qr_code, _svg = ac1_chat.get_qr_code()
-    ac2.secure_join(qr_code)
-
-    ac1.wait_for_securejoin_inviter_success()
-
-    ac1_new_chat = ac1.create_group("Another group")
-    ac1_new_chat.add_contact(ac1.get_contact_by_addr(ac2.get_config("addr")))
-    # Receive "Member added" message.
-    ac2.wait_for_incoming_msg_event()
-
-    ac1_new_chat.send_text("Hello!")
-    ac2_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert ac2_msg.text == "Hello!"
-    assert ac2_msg.chat.get_basic_snapshot().is_contact_request
-
-
-def test_aeap_flow_verified(acfactory):
-    """Test that a new address is added to a contact when it changes its address."""
-    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
-
-    logging.info("ac1: create verified-group QR, ac2 scans and joins")
-    chat = ac1.create_group("hello", protect=True)
-    assert chat.get_basic_snapshot().is_protected
-    qr_code, _svg = chat.get_qr_code()
-    logging.info("ac2: start QR-code based join-group protocol")
-    ac2.secure_join(qr_code)
-    ac1.wait_for_securejoin_inviter_success()
-
-    logging.info("sending first message")
-    msg_out = chat.send_text("old address").get_snapshot()
-
-    logging.info("receiving first message")
-    ac2.wait_for_incoming_msg_event()  # member added message
-    msg_in_1 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
-    assert msg_in_1.text == msg_out.text
-
-    logging.info("changing email account")
-    ac1.set_config("addr", ac1new.get_config("addr"))
-    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
-    ac1.stop_io()
-    ac1.configure()
-    ac1.start_io()
-
-    logging.info("sending second message")
-    msg_out = chat.send_text("changed address").get_snapshot()
-
-    logging.info("receiving second message")
-    msg_in_2 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
-    msg_in_2_snapshot = msg_in_2.get_snapshot()
-    assert msg_in_2_snapshot.text == msg_out.text
-    assert msg_in_2_snapshot.chat.id == msg_in_1.chat.id
-    assert msg_in_2.get_sender_contact().get_snapshot().address == ac1new.get_config("addr")
-    assert len(msg_in_2_snapshot.chat.get_contacts()) == 2
-    assert ac1new.get_config("addr") in [
-        contact.get_snapshot().address for contact in msg_in_2_snapshot.chat.get_contacts()
-    ]

deltachat-rpc-client/tests/test_something.py

@@ -358,22 +358,3 @@ def test_openrpc_command_line() -> None:
     openrpc = json.loads(out)
     assert "openrpc" in openrpc
     assert "methods" in openrpc
-
-
-def test_provider_info(rpc) -> None:
-    account_id = rpc.add_account()
-
-    provider_info = rpc.get_provider_info(account_id, "example.org")
-    assert provider_info["id"] == "example.com"
-
-    provider_info = rpc.get_provider_info(account_id, "uep7oiw4ahtaizuloith.org")
-    assert provider_info is None
-
-    # Test MX record resolution.
-    provider_info = rpc.get_provider_info(account_id, "github.com")
-    assert provider_info["id"] == "gmail"
-
-    # Disable MX record resolution.
-    rpc.set_config(account_id, "socks5_enabled", "1")
-    provider_info = rpc.get_provider_info(account_id, "github.com")
-    assert provider_info is None

deltachat-rpc-client/tox.ini

@@ -11,7 +11,7 @@ setenv =
 # Avoid stack overflow when Rust core is built without optimizations.
     RUST_MIN_STACK=8388608
 passenv =
-    CHATMAIL_DOMAIN
+    DCC_NEW_TMP_EMAIL
 deps =
     pytest
     pytest-timeout
@@ -28,6 +28,4 @@ commands =
     ruff src/ examples/ tests/
 
 [pytest]
-timeout = 300
-log_cli = true
-log_level = debug
+timeout = 60

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.131.6"
+version = "1.127.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

deny.toml

@@ -11,7 +11,6 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
-     { name = "async-channel", version = "1.9.0" },
      { name = "base16ct", version = "0.1.1" },
      { name = "base64", version = "<0.21" },
      { name = "bitflags", version = "1.3.2" },
@@ -25,18 +24,19 @@ skip = [
      { name = "digest", version = "<0.10" },
      { name = "ed25519-dalek", version = "1.0.1" },
      { name = "ed25519", version = "1.5.3" },
-     { name = "event-listener", version = "2.5.3" },
      { name = "getrandom", version = "<0.2" },
+     { name = "hashbrown", version = "<0.14.0" },
+     { name = "indexmap", version = "<2.0.0" },
      { name = "pem-rfc7468", version = "0.6.0" },
      { name = "pkcs8", version = "0.9.0" },
      { name = "quick-error", version = "<2.0" },
      { name = "rand_chacha", version = "<0.3" },
      { name = "rand_core", version = "<0.6" },
      { name = "rand", version = "<0.8" },
-     { name = "redox_syscall", version = "0.3.5" },
+     { name = "redox_syscall", version = "0.2.16" },
      { name = "regex-automata", version = "0.1.10" },
-     { name = "regex-syntax", version = "0.6.29" },
      { name = "ring", version = "0.16.20" },
+     { name = "regex-syntax", version = "0.6.29" },
      { name = "sec1", version = "0.3.0" },
      { name = "sha2", version = "<0.10" },
      { name = "signature", version = "1.6.4" },

draft/aeap-mvp.md

@@ -57,7 +57,7 @@ Note that usually a mail is signed by a key that has a UID matching the from add
 ### Notes:
   
 - We treat protected and non-protected chats the same
-- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more people know what old addresses you had).
+- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more pepole know what old addresses you had).
 - As soon as we encrypt read receipts, sending a read receipt will be enough to tell a lot of people that you transitioned
 - AEAP will make the problem of inconsistent group state worse, both because it doesn't work if the message is unencrypted (even if the design allowed it, it would be problematic security-wise) and because some chat partners may have gotten the transition and some not. We should do something against this at some point in the future, like asking the user whether they want to add/remove the members to restore consistent group state.
 

node/README.md

@@ -40,7 +40,7 @@ npm install deltachat-node
 
 ## Dependencies
 
-- Nodejs >= `v18.0.0`
+- Nodejs >= `v16.0.0`
 - rustup (optional if you can't use the prebuilds)
 
 > On Windows, you may need to also install **Perl** to be able to compile deltachat-core.
@@ -113,8 +113,8 @@ Then, in the `deltachat-desktop` repository, run:
 deltachat doesn't support universal (fat) binaries (that contain builds for both cpu architectures) yet, until it does you can use the following workaround to get x86_64 builds:
 
 ```
-$ fnm install 19 --arch x64
-$ fnm use 19
+$ fnm install 17 --arch x64
+$ fnm use 17
 $ node -p process.arch
 # result should be x64
 $ rustup target add x86_64-apple-darwin
@@ -127,8 +127,8 @@ $ npm run test
 If your node and electron are already build for arm64 you can also try building for arm:
 
 ```
-$ fnm install 18 --arch arm64
-$ fnm use 18
+$ fnm install 16 --arch arm64
+$ fnm use 16
 $ node -p process.arch
 # result should be arm64
 $ npm_config_arch=arm64 npm run build
@@ -204,10 +204,10 @@ Running `npm test` ends with showing a code coverage report, which is produced b
 
 The coverage report from `nyc` in the console is rather limited. To get a more detailed coverage report you can run `npm run coverage-html-report`. This will produce a html report from the `nyc` data and display it in a browser on your local machine.
 
-To run the integration tests you need to set the `CHATMAIL_DOMAIN` environment variables. E.g.:
+To run the integration tests you need to set the `DCC_NEW_TMP_EMAIL` environment variables. E.g.:
 
 ```
-$ export CHATMAIL_DOMAIN=chat.example.org
+$ export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=[token]
 $ npm run test
 ```
 

node/constants.js

@@ -28,7 +28,6 @@ module.exports = {
   DC_DOWNLOAD_DONE: 0,
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
-  DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
@@ -240,7 +239,6 @@ module.exports = {
   DC_STR_MSGGRPNAME: 15,
   DC_STR_MSGLOCATIONDISABLED: 65,
   DC_STR_MSGLOCATIONENABLED: 64,
-  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE: 172,
   DC_STR_NOMESSAGES: 1,
   DC_STR_NOT_CONNECTED: 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER: 113,

node/lib/constants.ts

@@ -28,7 +28,6 @@ export enum C {
   DC_DOWNLOAD_DONE = 0,
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
-  DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
@@ -240,7 +239,6 @@ export enum C {
   DC_STR_MSGGRPNAME = 15,
   DC_STR_MSGLOCATIONDISABLED = 65,
   DC_STR_MSGLOCATIONENABLED = 64,
-  DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE = 172,
   DC_STR_NOMESSAGES = 1,
   DC_STR_NOT_CONNECTED = 121,
   DC_STR_NOT_SUPPORTED_BY_PROVIDER = 113,

node/test/test.js

@@ -8,17 +8,26 @@ import { EventId2EventName, C } from '../dist/constants'
 import { join } from 'path'
 import { statSync } from 'fs'
 import { Context } from '../dist/context'
+import fetch from 'node-fetch'
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 
-function createTempUser(chatmailDomain) {
-  const charset = "2345789acdefghjkmnpqrstuvwxyz";
-  let user = "ci-";
-  for (let i = 0; i < 6; i++) {
-    user += charset[Math.floor(Math.random() * charset.length)];
+async function createTempUser(url) {
+  async function postData(url = '') {
+    // Default options are marked with *
+    const response = await fetch(url, {
+      method: 'POST', // *GET, POST, PUT, DELETE, etc.
+      headers: {
+        'cache-control': 'no-cache',
+      },
+    })
+    if (!response.ok) {
+      throw new Error('request failed: ' + response.body.read())
+    }
+    return response.json() // parses JSON response into native JavaScript objects
   }
-  const email = user + "@" + chatmailDomain;
-  return { email: email, password: user + "$" + user };
+
+  return await postData(url)
 }
 
 describe('static tests', function () {
@@ -667,9 +676,9 @@ describe('Offline Tests with unconfigured account', function () {
 
     const lot = chatList.getSummary(0)
     strictEqual(lot.getId(), 0, 'lot has no id')
-    strictEqual(lot.getState(), C.DC_STATE_IN_NOTICED, 'correct state')
+    strictEqual(lot.getState(), C.DC_STATE_UNDEFINED, 'correct state')
 
-    const text = 'Others will only see this group after you sent a first message.'
+    const text = 'No messages.'
     context.createGroupChat('groupchat1111')
     chatList = context.getChatList(0, 'groupchat1111', null)
     strictEqual(
@@ -759,7 +768,14 @@ describe('Integration tests', function () {
   })
 
   this.beforeAll(async function () {
-    account = createTempUser(process.env.CHATMAIL_DOMAIN)
+    if (!process.env.DCC_NEW_TMP_EMAIL) {
+      console.log(
+        'Missing DCC_NEW_TMP_EMAIL environment variable!, skip integration tests'
+      )
+      this.skip()
+    }
+
+    account = await createTempUser(process.env.DCC_NEW_TMP_EMAIL)
     if (!account || !account.email || !account.password) {
       console.log(
         "We didn't got back an account from the api, skip integration tests"

node/tsconfig.json

@@ -13,10 +13,10 @@
     },
     "exclude": ["node_modules", "deltachat-core-rust", "dist", "scripts"],
     "typedocOptions": {
+        "mode": "file",
         "out": "docs",
-        "excludePrivate": true,
+        "excludeNotExported": true,
         "defaultCategory": "index",
-        "includeVersion": true,
-        "entryPoints": ["lib/index.ts"]
+        "includeVersion": true
     }
 }

node/windows.md

@@ -6,7 +6,7 @@ E.g via <https://git-scm.com/download/win>
 
 ## install node
 
-Download and install `v18` from <https://nodejs.org/en/>
+Download and install `v16` from <https://nodejs.org/en/>
 
 ## install rust
 

package.json

@@ -2,25 +2,28 @@
   "dependencies": {
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
-    "node-gyp-build": "^4.6.1"
+    "node-gyp-build": "^4.1.0"
   },
   "description": "node.js bindings for deltachat-core",
   "devDependencies": {
     "@types/debug": "^4.1.7",
-    "@types/node": "^20.8.10",
+    "@types/node": "^16.11.26",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "esm": "^3.2.25",
+    "hallmark": "^2.0.0",
     "mocha": "^8.2.1",
-    "node-gyp": "^10.0.0",
-    "prebuildify": "^5.0.1",
-    "prebuildify-ci": "^1.0.5",
-    "prettier": "^3.0.3",
-    "typedoc": "^0.25.3",
-    "typescript": "^5.2.2"
+    "node-fetch": "^2.6.7",
+    "node-gyp": "^9.0.0",
+    "opn-cli": "^5.0.0",
+    "prebuildify": "^3.0.0",
+    "prebuildify-ci": "^1.0.4",
+    "prettier": "^2.0.5",
+    "typedoc": "^0.17.0",
+    "typescript": "^3.9.10"
   },
   "engines": {
-    "node": ">=18.0.0"
+    "node": ">=16.0.0"
   },
   "files": [
     "node/scripts/*",
@@ -46,15 +49,16 @@
     "build:core:rust": "node node/scripts/rebuild-core.js",
     "clean": "rm -rf node/dist node/build node/prebuilds node/node_modules ./target",
     "download-prebuilds": "prebuildify-ci download",
+    "hallmark": "hallmark --fix",
     "install": "node node/scripts/install.js",
     "install:prebuilds": "cd node && node-gyp-build \"npm run build:core\" \"npm run build:bindings:c:postinstall\"",
     "lint": "prettier --check \"node/lib/**/*.{ts,tsx}\"",
     "lint-fix": "prettier --write \"node/lib/**/*.{ts,tsx}\" \"node/test/**/*.js\"",
-    "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
+    "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.131.6"
+  "version": "1.127.0"
 }

python/README.rst

@@ -1,6 +1,6 @@
-============================
-CFFI Python Bindings
-============================
+=========================
+DeltaChat Python bindings
+=========================
 
 This package provides `Python bindings`_ to the `deltachat-core library`_
 which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
@@ -8,3 +8,157 @@ a low-level Chat/Contact/Message API to user interfaces and bots.
 
 .. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
 .. _`Python bindings`: https://py.delta.chat/
+
+Installing pre-built packages (Linux-only)
+==========================================
+
+If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
+
+We recommend to first create a fresh Python virtual environment
+and activate it in your shell::
+
+    python -m venv env
+    source env/bin/activate
+
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``env`` directory and leaves
+your system installation alone.
+
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
+
+    pip install deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
+
+Running tests
+=============
+
+Recommended way to run tests is using `scripts/run-python-test.sh`
+script provided in the core repository.
+
+This script compiles the library in debug mode and runs the tests using `tox`_.
+By default it will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _`tox`: https://tox.wiki
+.. _livetests:
+
+Running "live" tests with temporary accounts
+--------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
+
+Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
+
+    export DCC_NEW_TMP_EMAIL=<URL you got from us>
+
+With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts are removed automatically as they expire.
+After setting the variable, either rerun `scripts/run-python-test.sh`
+or run offline and online tests with `tox` directly::
+
+    tox -e py
+
+Each test run creates new accounts.
+
+Developing the bindings
+-----------------------
+
+If you want to develop or debug the bindings,
+you can create a testing development environment using `tox`::
+
+    export DCC_RS_DEV="$PWD"
+    export DCC_RS_TARGET=debug
+    tox -c python --devenv env -e py
+    . env/bin/activate
+
+Inside this environment the bindings are installed
+in editable mode (as if installed with `python -m pip install -e`)
+together with the testing dependencies like `pytest` and its plugins.
+
+You can then edit the source code in the development tree
+and quickly run `pytest` manually without waiting  for `tox`
+to recreating the virtual environment each time.
+
+.. _sourceinstall:
+
+Installing bindings from source
+===============================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+First, build the core library::
+
+   cargo build --release -p deltachat_ffi --features jsonrpc
+
+`jsonrpc` feature is required even if not used by the bindings
+because `deltachat.h` includes JSON-RPC functions unconditionally.
+
+Create the virtual environment and activate it:
+
+   python -m venv env
+   source env/bin/activate
+
+Build and install the bindings:
+
+   export DCC_RS_DEV="$PWD"
+   export DCC_RS_TARGET=release
+   python -m pip install ./python
+
+`DCC_RS_DEV` environment variable specifies the location of
+the core development tree. If this variable is not set,
+`libdeltachat` library and `deltachat.h` header are expected
+to be installed system-wide.
+
+When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
+the build profile name to look up the artifacts
+in the target directory.
+In this case setting it can be skipped because
+`DCC_RS_TARGET=release` is the default.
+
+Building manylinux based wheels
+===============================
+
+Building portable manylinux wheels which come with libdeltachat.so
+can be done with Docker_ or Podman_.
+
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
+
+If you want to build your own wheels, build container image first::
+
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
+
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
+find it with::
+
+   $ docker images
+
+This docker image can be used to run tests and build Python wheels for all interpreters::
+
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/doc/Makefile

@@ -0,0 +1,197 @@
+# Makefile for Sphinx documentation
+#
+
+VERSION       = $(shell python -c "import conf ; print(conf.version)")
+DOCZIP        = devpi-$(VERSION).doc.zip
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+RSYNCOPTS     = -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+
+export HOME=/tmp/home
+export TESTHOME=$(HOME)
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+# This variable is not auto generated as the order is important.
+USER_MAN_CHAPTERS = commands\
+                    user\
+                    indices\
+                    packages\
+#                 userman/index.rst\
+#                 userman/devpi_misc.rst\
+#                 userman/devpi_concepts.rst\
+
+
+#export DEVPI_CLIENTDIR=$(CURDIR)/.tmp_devpi_user_man/client
+#export DEVPI_SERVERDIR=$(CURDIR)/.tmp_devpi_user_man/server
+
+chapter = commands
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \
+        epub latex latexpdf text man changes linkcheck doctest gettext install \
+        quickstart-releaseprocess quickstart-pypimirror quickstart-server regen \
+        prepare-quickstart\
+        regen.server-fresh regen.server-restart regen.server-clean\
+        regen.uman-all regen.uman
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo
+	@echo "User Manual Regen Targets"
+	@echo "  regen.uman        regenerates page. of the user manual chapeter e.g. regen.uman chapter=..."
+	@echo "  regen.uman-all    regenerates the user manual"
+	@echo "  regen.uman-clean  stop temp server and clean up directory"
+	@echo "  Chapter List: $(USER_MAN_CHAPTERS)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+version:
+	@echo "version $(VERSION)"
+
+doczip: html
+	python doczip.py $(DOCZIP) _build/html
+
+install: html
+	rsync -avz $(RSYNCOPTS) _build/html/ delta@py.delta.chat:build/master
+
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/devpi.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/devpi.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/devpi"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/devpi"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+

python/doc/_templates/globaltoc.html

@@ -0,0 +1,17 @@
+
+<div class="globaltoc">
+
+<ul>
+  <li><a href="{{ pathto('index') }}">index</a></li>
+  <li><a href="{{ pathto('install') }}">install</a></li>
+  <li><a href="{{ pathto('api') }}">high level API</a></li>
+  <li><a href="{{ pathto('lapi') }}">low level API</a></li>
+</ul>
+<b>external links:</b>
+<ul>
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
+  <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
+  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
+</ul>
+
+</div>

python/doc/_templates/sidebarintro.html

@@ -0,0 +1 @@
+<h3>deltachat {{release}}</h3>

python/doc/api.rst

@@ -1,4 +1,5 @@
-High Level API Reference
+
+high level API reference
 ========================
 
 - :class:`deltachat.Account` (your main entry point, creates the
@@ -7,14 +8,28 @@ High Level API Reference
 - :class:`deltachat.Chat`
 - :class:`deltachat.Message`
 
+Account
+-------
+
 .. autoclass:: deltachat.Account
-   :members:
+    :members:
+
+
+Contact
+-------
 
 .. autoclass:: deltachat.Contact
-   :members:
+    :members:
+
+Chat
+----
 
 .. autoclass:: deltachat.Chat
-   :members:
+    :members:
+
+Message
+-------
 
 .. autoclass:: deltachat.Message
-   :members:
+    :members:
+

python/doc/cffi/install.rst

@@ -1,80 +0,0 @@
-Install
-=======
-
-Installing pre-built packages (Linux-only)
-------------------------------------------
-
-If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself`__.
-
-__ sourceinstall_
-
-We recommend to first create a fresh Python virtual environment
-and activate it in your shell::
-
-    python -m venv env
-    source env/bin/activate
-
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``env`` directory and leaves
-your system installation alone.
-
-For Linux we build wheels for all releases and push them to a python package
-index. To install the latest release::
-
-    pip install deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
-
-.. _sourceinstall:
-
-Installing bindings from source
--------------------------------
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-First, build the core library::
-
-   cargo build --release -p deltachat_ffi --features jsonrpc
-
-`jsonrpc` feature is required even if not used by the bindings
-because `deltachat.h` includes JSON-RPC functions unconditionally.
-
-Create the virtual environment and activate it::
-
-   python -m venv env
-   source env/bin/activate
-
-Build and install the bindings::
-
-   export DCC_RS_DEV="$PWD"
-   export DCC_RS_TARGET=release
-   python -m pip install ./python
-
-`DCC_RS_DEV` environment variable specifies the location of
-the core development tree. If this variable is not set,
-`libdeltachat` library and `deltachat.h` header are expected
-to be installed system-wide.
-
-When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
-the build profile name to look up the artifacts
-in the target directory.
-In this case setting it can be skipped because
-`DCC_RS_TARGET=release` is the default.

python/doc/cffi/intro.rst

@@ -1,11 +0,0 @@
-Introduction
-============
-
-CFFI bindings are available via the `deltachat <https://pypi.org/project/deltachat/>`_ Python package.
-The package contains both the Python bindings and the Delta Chat core.
-It is provided only for Linux.
-
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
-low-level CFFI bindings to the C interface of the Delta Chat core
-and high-level Python bindings built on top of CFFI bindings.

python/doc/cffi/manylinux.rst

@@ -1,25 +0,0 @@
-Building Manylinux-Based Wheels
-===============================
-
-Building portable manylinux wheels which come with libdeltachat.so
-can be done with Docker_ or Podman_.
-
-.. _Docker: https://www.docker.com/
-.. _Podman: https://podman.io/
-
-If you want to build your own wheels, build container image first::
-
-   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
-   $ docker build -t deltachat/coredeps scripts/coredeps
-
-This will use the ``scripts/coredeps/Dockerfile`` to build
-container image called ``deltachat/coredeps``.  You can afterwards
-find it with::
-
-   $ docker images
-
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e CHATMAIL_DOMAIN \
-       --rm -it -v $(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh

python/doc/cffi/tests.rst

@@ -1,49 +0,0 @@
-Running Tests
-=============
-
-Recommended way to run tests is using `scripts/run-python-test.sh`
-script provided in the core repository.
-
-This script compiles the library in debug mode and runs the tests using `tox`_.
-By default it will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real email servers.
-
-.. _`tox`: https://tox.wiki
-.. _livetests:
-
-Running "Live" Tests With Temporary Accounts
---------------------------------------------
-
-If you want to run live functional tests
-you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
-that creates email accounts like this::
-
-    export CHATMAIL_DOMAIN=nine.testrun.org
-
-With this account-creation setting, pytest runs create ephemeral email accounts on the server.
-These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
-After setting the variable, either rerun `scripts/run-python-test.sh`
-or run offline and online tests with `tox` directly::
-
-    tox -e py
-
-Each test run creates new accounts.
-
-Developing the Bindings
------------------------
-
-If you want to develop or debug the bindings,
-you can create a testing development environment using `tox`::
-
-    export DCC_RS_DEV="$PWD"
-    export DCC_RS_TARGET=debug
-    tox -c python --devenv env -e py
-    . env/bin/activate
-
-Inside this environment the bindings are installed
-in editable mode (as if installed with `python -m pip install -e`)
-together with the testing dependencies like `pytest` and its plugins.
-
-You can then edit the source code in the development tree
-and quickly run `pytest` manually without waiting  for `tox`
-to recreating the virtual environment each time.

python/doc/changelog.rst

@@ -0,0 +1,4 @@
+Changelog for deltachat-core's Python bindings
+==============================================
+
+.. include:: ../CHANGELOG

python/doc/conf.py

@@ -1,94 +1,138 @@
-from pathlib import Path
+# -*- coding: utf-8 -*-
+#
+# devpi documentation build configuration file, created by
+# sphinx-quickstart on Mon Jun  3 16:11:22 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 from deltachat import __version__ as release
-
 version = ".".join(release.split(".")[:2])
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-# sys.path.insert(0, os.path.abspath('.'))
+#sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx.ext.autosummary",
-    "sphinx.ext.viewcode",
-    "breathe",
-    "sphinx_rtd_theme",
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    #'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.viewcode',
+    'breathe',
+    #'sphinx.ext.githubpages',
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
+templates_path = ['_templates']
 
 # The suffix of source filenames.
-source_suffix = ".rst"
+source_suffix = '.rst'
 
 # The encoding of source files.
-# source_encoding = 'utf-8-sig'
+#source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = "index"
+master_doc = 'index'
 
 # General information about the project.
-project = "Delta Chat"
-copyright = "2023, Delta Chat contributors"
-author = "Delta Chat contributors"
+project = u'deltachat'
+copyright = u'2020, holger krekel and contributors'
 
 
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-# today = ''
+#today = ''
 # Else, today_fmt is used as the format for a strftime call.
-# today_fmt = '%B %d, %Y'
+#today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ["sketch", "_build", "attic", "Thumbs.db", ".DS_Store"]
+exclude_patterns = ['sketch', '_build', "attic"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-# default_role = None
+#default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-# add_function_parentheses = True
+#add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-# add_module_names = True
+#add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-# show_authors = False
+#show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = "sphinx"
+pygments_style = 'sphinx'
 
 # A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
+#modindex_common_prefix = []
 
 
 # -- breathe options ------
 
-breathe_projects = {"deltachat": Path("../../docs/xml/")}
+breathe_projects = {
+    "deltachat": "../../docs/xml/"
+}
 
 breathe_default_project = "deltachat"
 
 # -- Options for HTML output ---------------------------------------------------
 
-html_theme = "sphinx_rtd_theme"
+sys.path.append(os.path.abspath('_themes'))
+html_theme_path = ['_themes']
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+# html_theme = 'flask'
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+    'logo': '_static/delta-chat.svg',
+    'font_size': "1.1em",
+    'caption_font_size': "0.9em",
+    'code_font_size': "1.1em",
+
+
+}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = ["_themes"]
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-# html_title = None
+#html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-# html_short_title = None
+#html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
@@ -97,34 +141,51 @@ html_logo = "_static/delta-chat.svg"
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = "_static/favicon.ico"
+html_favicon = '_static/favicon.ico'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
+html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-# html_last_updated_fmt = '%b %d, %Y'
+#html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-# html_use_smartypants = True
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#
+html_sidebars = {
+    'index': [
+        'sidebarintro.html',
+        'globaltoc.html',
+        'searchbox.html'
+    ],
+    '**': [
+        'sidebarintro.html',
+        'globaltoc.html',
+        'relations.html',
+        'searchbox.html'
+    ]
+}
+
 
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-# html_additional_pages = {}
+#html_additional_pages = {}
 
 # If false, no module index is generated.
-# html_domain_indices = True
+#html_domain_indices = True
 
 # If false, no index is generated.
-# html_use_index = True
+#html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-# html_split_index = False
+#html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
@@ -133,65 +194,71 @@ html_show_sourcelink = False
 html_show_sphinx = False
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-# html_show_copyright = True
+#html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-html_use_opensearch = "https://doc.devpi.net"
+html_use_opensearch = 'https://doc.devpi.net'
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
+#html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = "deltachat-python"
+htmlhelp_basename = 'deltachat-python'
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #'pointsize': '12pt',
-    # Additional stuff for the LaTeX preamble.
-    #'preamble': '',
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+'pointsize': '12pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ("index", "devpi.tex", "deltachat documentation", "holger krekel", "manual"),
+  ('index', 'devpi.tex', u'deltachat documentation',
+   u'holger krekel', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-# latex_logo = None
+#latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-# latex_use_parts = False
+#latex_use_parts = False
 
 # If true, show page references after internal links.
-# latex_show_pagerefs = False
+#latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-# latex_show_urls = False
+#latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-# latex_appendices = []
+#latex_appendices = []
 
 # If false, no module index is generated.
-# latex_domain_indices = True
+#latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [("index", "deltachat", "deltachat documentation", ["holger krekel"], 1)]
+man_pages = [
+    ('index', 'deltachat', u'deltachat documentation',
+     [u'holger krekel'], 1)
+]
 
 # If true, show URL addresses after external links.
-# man_show_urls = False
+#man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -200,38 +267,30 @@ man_pages = [("index", "deltachat", "deltachat documentation", ["holger krekel"]
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (
-        "index",
-        "devpi",
-        "devpi Documentation",
-        "holger krekel",
-        "devpi",
-        "One line description of project.",
-        "Miscellaneous",
-    ),
+  ('index', 'devpi', u'devpi Documentation',
+   u'holger krekel', 'devpi', 'One line description of project.',
+   'Miscellaneous'),
 ]
 
 # Documents to append as an appendix to all manuals.
-# texinfo_appendices = []
+#texinfo_appendices = []
 
 # If false, no module index is generated.
-# texinfo_domain_indices = True
+#texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-# texinfo_show_urls = 'footnote'
+#texinfo_show_urls = 'footnote'
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {"http://docs.python.org/": None}
+intersphinx_mapping = {'http://docs.python.org/': None}
 
 # autodoc options
 autodoc_member_order = "bysource"
-
-
 # always document __init__ functions
 def skip(app, what, name, obj, skip, options):
     return skip
 
-
 def setup(app):
     app.connect("autodoc-skip-member", skip)
+

python/doc/examples.rst

@@ -1,10 +1,11 @@
-Examples
+
+examples
 ========
 
 Once you have :doc:`installed deltachat bindings <install>`
 you need email/password credentials for an IMAP/SMTP account.
 Delta Chat developers and the CI system use a special URL to create
-temporary email accounts on `testrun.org <https://testrun.org/>`_ for testing.
+temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
 Receiving a Chat message from the command line
 ----------------------------------------------
@@ -15,11 +16,11 @@ Here is a simple bot that:
 
 - terminates the bot if the message `/quit` is sent
 
-.. include:: ../../examples/echo_and_quit.py
+.. include:: ../examples/echo_and_quit.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an email address and password of
+by specifying a database path, an e-mail address and password of
 a SMTP-IMAP account::
 
     $ cd examples
@@ -39,11 +40,11 @@ Here is a simple bot that:
 
 - tracks member additions and removals for all chat groups
 
-.. include:: ../../examples/group_tracking.py
+.. include:: ../examples/group_tracking.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an email address and password of
+by specifying a database path, an e-mail address and password of
 a SMTP-IMAP account::
 
     python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db

python/doc/index.rst

@@ -1,44 +1,41 @@
-Delta Chat Python bindings, new and old
-=======
+deltachat python bindings
+=========================
 
-`Delta Chat <https://delta.chat/>`_ provides two kinds of Python bindings
-to the `Rust Core <https://github.com/deltachat/deltachat-core-rust>`_:
-JSON-RPC bindings and CFFI bindings.
-When starting a new project it is recommended to use JSON-RPC bindings,
-which are used in the Delta Chat Desktop app through generated Typescript-bindings. 
-The Python JSON-RPC bindings are maintained by Delta Chat core developers. 
-Most existing bot projects and many tests in Delta Chat's own core library
-still use the CFFI-bindings, and it is going to be maintained certainly also in 2024. 
-New APIs might however only appear in the JSON-RPC bindings, 
-as the CFFI bindings are increasingly in maintenance-only mode. 
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
+
+- :doc:`api` is a high level interface to deltachat-core.
+
+- :doc:`plugins` is a brief introduction into implementing plugin hooks.
+
+- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
+  <https://github.com/deltachat/deltachat-core-rust>`_.
+
+
+
+getting started
+---------------
 
 .. toctree::
    :maxdepth: 2
-   :caption: JSON-RPC Bindings
 
-   jsonrpc/intro
-   jsonrpc/install
-   jsonrpc/examples
-   jsonrpc/reference
-   jsonrpc/develop
+   install
+   examples
 
 .. toctree::
-   :maxdepth: 2
-   :caption: CFFI Bindings
+   :hidden:
 
-   cffi/intro
-   cffi/install
-   cffi/examples
-   cffi/manylinux
-   cffi/tests
-   cffi/api
-   cffi/lapi
-   cffi/plugins
+   links
+   changelog
+   api
+   lapi
+   plugins
+
+..
+    Indices and tables
+    ==================
+
+    * :ref:`genindex`
+    * :ref:`modindex`
+    * :ref:`search`
 
-.. _`deltachat`: https://delta.chat
-.. _`deltachat-core repo`: https://github.com/deltachat
-.. _pip: http://pypi.org/project/pip/
-.. _virtualenv: http://pypi.org/project/virtualenv/
-.. _merlinux: http://merlinux.eu
-.. _pypi: http://pypi.org/
-.. _`issue-tracker`: https://github.com/deltachat/deltachat-core-rust

python/doc/install.rst

@@ -0,0 +1,2 @@
+
+.. include:: ../README.rst

python/doc/jsonrpc/develop.rst

@@ -1,68 +0,0 @@
-===========
-Development
-===========
-
-To develop JSON-RPC bindings,
-clone the `deltachat-core-rust <https://github.com/deltachat/deltachat-core-rust/>`_ repository::
-
-   git clone https://github.com/deltachat/deltachat-core-rust.git
-
-Testing
-=======
-
-To run online tests, set ``CHATMAIL_DOMAIN``
-to a domain of the email server
-that can be used to create testing accounts::
-
-    export CHATMAIL_DOMAIN=nine.testrun.org
-
-Then run ``scripts/run-rpc-test.sh``
-to build debug version of ``deltachat-rpc-server``
-and run ``deltachat-rpc-client`` tests
-in a separate virtual environment managed by `tox <https://tox.wiki/>`_.
-
-Development Environment
-=======================
-
-Creating a new virtual environment
-to run the tests each time
-as ``scripts/run-rpc-test.sh`` does is slow
-if you are changing the tests or the code
-and want to rerun the tests each time.
-
-If you are developing the tests,
-it is better to create a persistent virtual environment.
-You can do this by running ``scripts/make-rpc-testenv.sh``.
-This creates a virtual environment ``venv`` which you can then enter with::
-
-   . venv/bin/activate
-
-Then you can run the tests with
-
-::
-
-    pytest deltachat-rpc-client/tests/
-
-Refer to `pytest documentation <https://docs.pytest.org/>` for details.
-
-If make the changes to Delta Chat core
-or Python bindings, you can rebuild the environment by rerunning
-``scripts/make-rpc-testenv.sh``.
-It is ok to rebuild the activated environment this way,
-you do not need to deactivate or reactivate the environment each time.
-
-Using REPL
-==========
-
-Once you have a development environment,
-you can quickly test things in REPL::
-
-   $ python
-   >>> from deltachat_rpc_client import *
-   >>> rpc = Rpc()
-   >>> rpc.start()
-   >>> dc = DeltaChat(rpc)
-   >>> system_info = dc.get_system_info()
-   >>> system_info["level"]
-   'awesome'
-   >>> rpc.close()

python/doc/jsonrpc/examples.rst

@@ -1,19 +0,0 @@
-Examples
-========
-
-Echo bot
---------
-.. include:: ../../../deltachat-rpc-client/examples/echobot_no_hooks.py
-    :literal:
-
-Echo bot with hooks
--------------------
-.. include:: ../../../deltachat-rpc-client/examples/echobot.py
-    :literal:
-
-Advanced echo bot
------------------
-
-.. include:: ../../../deltachat-rpc-client/examples/echobot_advanced.py
-    :literal:
-

python/doc/jsonrpc/install.rst

@@ -1,36 +0,0 @@
-Install
-=======
-
-To use JSON-RPC bindings for Delta Chat core you will need
-a ``deltachat-rpc-server`` binary which provides Delta Chat core API over JSON-RPC
-and a ``deltachat-rpc-client`` Python package which is a JSON-RPC client that starts ``deltachat-rpc-server`` process and uses JSON-RPC API.
-
-`Create a virtual environment <https://docs.python.org/3/library/venv.html>`__ if you
-don’t have one already and activate it::
-
-   $ python -m venv venv
-   $ . venv/bin/activate
-
-Install ``deltachat-rpc-server``
---------------------------------
-
-To get ``deltachat-rpc-server`` binary you have three options:
-
-1. Install ``deltachat-rpc-server`` from PyPI using ``pip install deltachat-rpc-server``.
-2. Build and install ``deltachat-rpc-server`` from source with ``cargo install --git https://github.com/deltachat/deltachat-core-rust/ deltachat-rpc-server``.
-3. Download prebuilt release from https://github.com/deltachat/deltachat-core-rust/releases and install it into ``PATH``.
-
-Check that ``deltachat-rpc-server`` is installed and can run::
-
-   $ deltachat-rpc-server --version
-   1.131.4
-
-Then install ``deltachat-rpc-client`` with ``pip install deltachat-rpc-client``.
-
-Install ``deltachat-rpc-client``
---------------------------------
-
-To get ``deltachat-rpc-client`` Python library you can:
-
-1. Install ``deltachat-rpc-client`` from PyPI using ``pip install deltachat-rpc-client``.
-2. Install ``deltachat-rpc-client`` from source with ``pip install git+https://github.com/deltachat/deltachat-core-rust.git@main#subdirectory=deltachat-rpc-client``.

python/doc/jsonrpc/intro.rst

@@ -1,8 +0,0 @@
-Introduction
-============
-
-JSON-RPC bindings are available via the `deltachat-rpc-client <https://pypi.org/project/deltachat-rpc-client/>`_ Python package.
-This package provides only the Python bindings and requires ``deltachat-rpc-server`` binary to be installed.
-`deltachat-rpc-server <https://pypi.org/project/deltachat-rpc-server/>`_ package provides ``deltachat-rpc-server`` binary for Linux, Windows, macOS and Android.
-
-RPC client connects to standalone Delta Chat RPC server ``deltachat-rpc-server`` and provides Python interface to it.

python/doc/jsonrpc/reference.rst

@@ -1,5 +0,0 @@
-API Reference
-=============
-
-.. automodule:: deltachat_rpc_client
-   :members:

python/doc/lapi.rst

@@ -1,7 +1,8 @@
-Low Level API Reference
-=======================
 
-For full doxygen-generated C-docs, defines and functions please checkout
+low level API reference
+===================================
+
+for full doxygen-generated C-docs, defines and functions please checkout
 
 https://c.delta.chat
 

python/doc/links.rst

@@ -0,0 +1,11 @@
+
+links
+================================
+
+.. _`deltachat`: https://delta.chat
+.. _`deltachat-core repo`: https://github.com/deltachat
+.. _pip: http://pypi.org/project/pip/
+.. _virtualenv: http://pypi.org/project/virtualenv/
+.. _merlinux: http://merlinux.eu
+.. _pypi: http://pypi.org/
+.. _`issue-tracker`: https://github.com/deltachat/deltachat-core

python/doc/make.bat

@@ -0,0 +1,190 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+	:help
+	echo.Please use `make ^<target^>` where ^<target^> is one of
+	echo.  html       to make standalone HTML files
+	echo.  dirhtml    to make HTML files named index.html in directories
+	echo.  singlehtml to make a single large HTML file
+	echo.  pickle     to make pickle files
+	echo.  json       to make JSON files
+	echo.  htmlhelp   to make HTML files and a HTML help project
+	echo.  qthelp     to make HTML files and a qthelp project
+	echo.  devhelp    to make HTML files and a Devhelp project
+	echo.  epub       to make an epub
+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+	echo.  text       to make text files
+	echo.  man        to make manual pages
+	echo.  texinfo    to make Texinfo files
+	echo.  gettext    to make PO message catalogs
+	echo.  changes    to make an overview over all changed/added/deprecated items
+	echo.  linkcheck  to check all external links for integrity
+	echo.  doctest    to run all doctests embedded in the documentation if enabled
+	goto end
+)
+
+if "%1" == "clean" (
+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+	del /q /s %BUILDDIR%\*
+	goto end
+)
+
+if "%1" == "html" (
+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	goto end
+)
+
+if "%1" == "dirhtml" (
+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+	goto end
+)
+
+if "%1" == "singlehtml" (
+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+	goto end
+)
+
+if "%1" == "pickle" (
+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the pickle files.
+	goto end
+)
+
+if "%1" == "json" (
+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the JSON files.
+	goto end
+)
+
+if "%1" == "htmlhelp" (
+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+	goto end
+)
+
+if "%1" == "qthelp" (
+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\devpi.qhcp
+	echo.To view the help file:
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\devpi.ghc
+	goto end
+)
+
+if "%1" == "devhelp" (
+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished.
+	goto end
+)
+
+if "%1" == "epub" (
+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub file is in %BUILDDIR%/epub.
+	goto end
+)
+
+if "%1" == "latex" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "text" (
+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The text files are in %BUILDDIR%/text.
+	goto end
+)
+
+if "%1" == "man" (
+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+	goto end
+)
+
+if "%1" == "texinfo" (
+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+	goto end
+)
+
+if "%1" == "gettext" (
+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+	goto end
+)
+
+if "%1" == "changes" (
+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.The overview file is in %BUILDDIR%/changes.
+	goto end
+)
+
+if "%1" == "linkcheck" (
+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+	goto end
+)
+
+if "%1" == "doctest" (
+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+	goto end
+)
+
+:end

python/src/deltachat/account.py

@@ -571,11 +571,11 @@ class Account:
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
-        """setup contact and return a `Chat` instance after contact is established.
-
-        This function triggers a network protocol in the background between
-        the emitter of the QR code and this account.
+        """setup contact and return a Chat after contact is established.
 
+        Note that this function may block for a long time as messages are exchanged
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
+        is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
         assert self.check_qr(qr).is_ask_verifycontact()
@@ -585,11 +585,11 @@ class Account:
         return Chat(self, chat_id)
 
     def qr_join_chat(self, qr):
-        """return a `Chat` instance for which the securejoin network
-        protocol has been started.
+        """join a chat group through a QR code.
 
-        This function triggers a network protocol in the background between
-        the emitter of the QR code and this account.
+        Note that this function may block for a long time as messages are exchanged
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
+        is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
         """

python/src/deltachat/testplugin.py

@@ -8,11 +8,11 @@ import sys
 import threading
 import time
 import weakref
-import random
 from queue import Queue
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
+import requests
 from _pytest._code import Source
 
 import deltachat
@@ -24,10 +24,10 @@ from .events import FFIEventLogger, FFIEventTracker
 def pytest_addoption(parser):
     group = parser.getgroup("deltachat testplugin options")
     group.addoption(
-        "--chatmail",
+        "--liveconfig",
         action="store",
         default=None,
-        help="chatmail server domain name",
+        help="a file with >=2 lines where each line contains NAME=VALUE config settings for one account",
     )
     group.addoption(
         "--ignored",
@@ -52,11 +52,11 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config):
-    cfg = config.getoption("--chatmail")
+    cfg = config.getoption("--liveconfig")
     if not cfg:
-        cfg = os.getenv("CHATMAIL_DOMAIN")
+        cfg = os.getenv("DCC_NEW_TMP_EMAIL")
         if cfg:
-            config.option.chatmail = cfg
+            config.option.liveconfig = cfg
 
     # Make sure we don't get garbled output because threads keep running
     # collect all ever created accounts in a weakref-set (so we don't
@@ -125,10 +125,13 @@ def pytest_report_header(config, startdir):
         ),
     ]
 
-    chatmail_opt = config.getoption("--chatmail")
-    if chatmail_opt:
-        summary.append(f"Chatmail account provider: {chatmail_opt}")
-
+    cfg = config.option.liveconfig
+    if cfg:
+        if "?" in cfg:
+            url, token = cfg.split("?", 1)
+            summary.append(f"Liveconfig provider: {url}?<token omitted>")
+        else:
+            summary.append(f"Liveconfig file: {cfg}")
     return summary
 
 
@@ -153,29 +156,38 @@ class TestProcess:
     def get_liveconfig_producer(self):
         """provide live account configs, cached on a per-test-process scope
         so that test functions can re-use already known live configs.
+        Depending on the --liveconfig option this comes from
+        a HTTP provider or a file with a line specifying each accounts config.
         """
-        chatmail_opt = self.pytestconfig.getoption("--chatmail")
-        if chatmail_opt:
-            # Use a chatmail instance.
-            domain = chatmail_opt
+        liveconfig_opt = self.pytestconfig.getoption("--liveconfig")
+        if not liveconfig_opt:
+            pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig to provide live accounts")
+
+        if not liveconfig_opt.startswith("http"):
+            for line in open(liveconfig_opt):
+                if line.strip() and not line.strip().startswith("#"):
+                    d = {}
+                    for part in line.split():
+                        name, value = part.split("=")
+                        d[name] = value
+                    self._configlist.append(d)
+
+            yield from iter(self._configlist)
+        else:
             MAX_LIVE_CREATED_ACCOUNTS = 10
             for index in range(MAX_LIVE_CREATED_ACCOUNTS):
                 try:
                     yield self._configlist[index]
                 except IndexError:
-                    part = "".join(random.choices("2345789acdefghjkmnpqrstuvwxyz", k=6))
-                    username = f"ci-{part}"
-                    password = f"{username}${username}"
-                    addr = f"{username}@{domain}"
-                    config = {"addr": addr, "mail_pw": password}
+                    res = requests.post(liveconfig_opt, timeout=60)
+                    if res.status_code != 200:
+                        pytest.fail(f"newtmpuser count={index} code={res.status_code}: '{res.text}'")
+                    d = res.json()
+                    config = {"addr": d["email"], "mail_pw": d["password"]}
                     print("newtmpuser {}: addr={}".format(index, config["addr"]))
                     self._configlist.append(config)
                     yield config
             pytest.fail(f"more than {MAX_LIVE_CREATED_ACCOUNTS} live accounts requested.")
-        else:
-            pytest.skip(
-                "specify CHATMAIL_DOMAIN or --chatmail to provide live accounts",
-            )
 
     def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path):
         db_target_path = pathlib.Path(db_target_path)
@@ -521,7 +533,6 @@ class ACFactory:
         configdict.setdefault("bcc_self", False)
         configdict.setdefault("mvbox_move", False)
         configdict.setdefault("sentbox_watch", False)
-        configdict.setdefault("sync_msgs", False)
         ac.update_config(configdict)
         self._preconfigure_key(ac, configdict["addr"])
         return ac

python/tests/test_0_complex_or_slow.py

@@ -140,10 +140,6 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
     assert msg.is_system_message()
     assert "added" in msg.text.lower()
 
-    assert any(
-        m.is_system_message() and m.text == "Messages are guaranteed to be end-to-end encrypted from now on."
-        for m in msg.chat.get_messages()
-    )
     lp.sec("ac1: send message")
     msg_out = chat1.send_text("hello")
     assert msg_out.is_encrypted()
@@ -584,7 +580,6 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.get_sender_contact().addr == ac1.get_config("addr")
     chat2 = msg_in.chat
     assert chat2.is_protected()
-    assert chat2.get_messages()[0].text == "Messages are guaranteed to be end-to-end encrypted from now on."
 
     lp.sec("ac2_offl: sending message")
     msg_out = chat2.send_text("hello")
@@ -652,15 +647,6 @@ def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
     chat2.send_text("hi2")
 
     lp.sec("ac2_offl: receiving message")
-    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-    msg_in = ac2_offl.get_message_by_id(ev.data2)
-    assert msg_in.is_system_message()
-    assert msg_in.text == "Messages are guaranteed to be end-to-end encrypted from now on."
-
-    # We need to consume one event that has data2=0
-    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-    assert ev.data2 == 0
-
     ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
     msg_in = ac2_offl.get_message_by_id(ev.data2)
     assert not msg_in.is_system_message()

python/tests/test_1_online.py

@@ -9,7 +9,7 @@ import pytest
 from imap_tools import AND, U
 
 import deltachat as dc
-from deltachat import account_hookimpl, Message
+from deltachat import account_hookimpl, Message, Chat
 from deltachat.tracker import ImexTracker
 
 
@@ -367,7 +367,7 @@ def test_webxdc_download_on_demand(acfactory, data, lp):
 
     lp.sec("ac2 sets download limit")
     ac2.set_config("download_limit", "100")
-    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
+    assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(50000))}, "some test data")
     ac2_update = ac2._evtracker.wait_next_incoming_message()
     assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
     assert not msg2.get_status_updates()
@@ -489,7 +489,7 @@ def test_forward_messages(acfactory, lp):
     lp.sec("ac2: check new chat has a forwarded message")
     assert chat3.is_promoted()
     messages = chat3.get_messages()
-    assert len(messages) == 2
+    assert len(messages) == 1
     msg = messages[-1]
     assert msg.is_forwarded()
     ac2.delete_messages(messages)
@@ -1445,7 +1445,7 @@ def test_reaction_to_partially_fetched_msg(acfactory, lp, tmp_path):
       message, then processes a partially downloaded message.
     - As a result, Bob does not see a reaction
     """
-    download_limit = 300000
+    download_limit = 32768
     ac1, ac2 = acfactory.get_online_accounts(2)
     ac1_addr = ac1.get_config("addr")
     chat = ac1.create_chat(ac2)
@@ -1658,6 +1658,75 @@ def test_ac_setup_message_twice(acfactory, lp):
     assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
 
+def test_qr_setup_contact(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
+    qr = ac1.get_setup_contact_qr()
+
+    lp.sec("ac2: start QR-code based setup contact protocol")
+    ch = ac2.qr_setup_contact(qr)
+    assert ch.id >= 10
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+
+@pytest.mark.parametrize("verified_one_on_one_chats", [0, 1])
+def test_qr_join_chat(acfactory, lp, verified_one_on_one_chats):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
+    ac2.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
+
+    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
+    chat = ac1.create_group_chat("hello")
+    qr = chat.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    ch = ac2.qr_join_chat(qr)
+    lp.sec("ac2: qr_join_chat() returned")
+    assert ch.id >= 10
+    # check that at least some of the handshake messages are deleted
+    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    msg = ac2._evtracker.wait_next_incoming_message()
+    assert msg.text == "Member Me ({}) added by {}.".format(ac2.get_config("addr"), ac1.get_config("addr"))
+
+    # ac1 reloads the chat.
+    chat = Chat(chat.account, chat.id)
+    assert not chat.is_protected()
+
+    # ac2 reloads the chat.
+    ch = Chat(ch.account, ch.id)
+    assert not ch.is_protected()
+
+
+def test_qr_new_group_unblocked(acfactory, lp):
+    """Regression test for a bug intoduced in core v1.113.0.
+    ac2 scans a verified group QR code created by ac1.
+    This results in creation of a blocked 1:1 chat with ac1 on ac2,
+    but ac1 contact is not blocked on ac2.
+    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
+    ac2 should receive a message and create a contact request for the group.
+    Due to a bug previously ac2 created a blocked group.
+    """
+
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1_chat = ac1.create_group_chat("Group for joining", verified=True)
+    qr = ac1_chat.get_join_qr()
+    ac2.qr_join_chat(qr)
+
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    ac1_new_chat = ac1.create_group_chat("Another group")
+    ac1_new_chat.add_contact(ac2)
+    # Receive "Member added" message.
+    ac2._evtracker.wait_next_incoming_message()
+
+    ac1_new_chat.send_text("Hello!")
+    ac2_msg = ac2._evtracker.wait_next_incoming_message()
+    assert ac2_msg.text == "Hello!"
+    assert ac2_msg.chat.is_contact_request()
+
+
 def test_qr_email_capitalization(acfactory, lp):
     """Regression test for a bug
     that resulted in failure to propagate verification via gossip in a verified group
@@ -1835,7 +1904,7 @@ def test_system_group_msg_from_blocked_user(acfactory, lp):
     chat_on_ac2.send_text("This will arrive")
     msg = ac1._evtracker.wait_next_incoming_message()
     assert msg.text == "This will arrive"
-    message_texts = [m.text for m in chat_on_ac1.get_messages() if not m.is_system_message()]
+    message_texts = [m.text for m in chat_on_ac1.get_messages()]
     assert len(message_texts) == 2
     assert "First group message" in message_texts
     assert "This will arrive" in message_texts
@@ -2139,17 +2208,7 @@ def test_delete_multiple_messages(acfactory, lp):
 
 def test_trash_multiple_messages(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
-
-    # Create the Trash folder on IMAP server
-    # and recreate the account so Trash folder is configured.
-    lp.sec("Creating trash folder")
-    ac2.direct_imap.create_folder("Trash")
-    lp.sec("Creating new accounts")
-    ac2 = acfactory.new_online_configuring_account(cloned_from=ac2)
-    acfactory.bring_accounts_online()
-
     ac2.set_config("delete_to_trash", "1")
-    assert ac2.get_config("configured_trash_folder")
     chat12 = acfactory.get_accepted_chat(ac1, ac2)
 
     lp.sec("ac1: sending 3 messages")
@@ -2180,15 +2239,13 @@ def test_trash_multiple_messages(acfactory, lp):
 
 
 def test_configure_error_msgs_wrong_pw(acfactory):
-    (ac1,) = acfactory.get_online_accounts(1)
-
-    ac2 = acfactory.get_unconfigured_account()
-    ac2.set_config("addr", ac1.get_config("addr"))
-    ac2.set_config("mail_pw", "abc")  # Wrong mail pw
-    ac2.configure()
+    configdict = acfactory.get_next_liveconfig()
+    ac1 = acfactory.get_unconfigured_account()
+    ac1.update_config(configdict)
+    ac1.set_config("mail_pw", "abc")  # Wrong mail pw
+    ac1.configure()
     while True:
-        ev = ac2._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        print(f"Configuration progress: {ev.data1}")
+        ev = ac1._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
         if ev.data1 == 0:
             break
     # Password is wrong so it definitely has to say something about "password"
@@ -2405,6 +2462,47 @@ def test_delete_deltachat_folder(acfactory):
     assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
+def test_aeap_flow_verified(acfactory, lp):
+    """Test that a new address is added to a contact when it changes its address."""
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+
+    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
+    chat = ac1.create_group_chat("hello", verified=True)
+    assert chat.is_protected()
+    qr = chat.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    chat2 = ac2.qr_join_chat(qr)
+    assert chat2.id >= 10
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    lp.sec("sending first message")
+    msg_out = chat.send_text("old address")
+
+    lp.sec("receiving first message")
+    ac2._evtracker.wait_next_incoming_message()  # member added message
+    msg_in_1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in_1.text == msg_out.text
+
+    lp.sec("changing email account")
+    ac1.set_config("addr", ac1new.get_config("addr"))
+    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
+    ac1.stop_io()
+    configtracker = ac1.configure()
+    configtracker.wait_finish()
+    ac1.start_io()
+
+    lp.sec("sending second message")
+    msg_out = chat.send_text("changed address")
+
+    lp.sec("receiving second message")
+    msg_in_2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in_2.text == msg_out.text
+    assert msg_in_2.chat.id == msg_in_1.chat.id
+    assert msg_in_2.get_sender_contact().addr == ac1new.get_config("addr")
+    assert len(msg_in_2.chat.get_contacts()) == 2
+    assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()]
+
+
 def test_archived_muted_chat(acfactory, lp):
     """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for
     DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously.
@@ -2450,7 +2548,7 @@ class TestOnlineConfigureFails:
     def test_invalid_user(self, acfactory):
         configdict = acfactory.get_next_liveconfig()
         ac1 = acfactory.get_unconfigured_account()
-        configdict["addr"] = "$" + configdict["addr"]
+        configdict["addr"] = "x" + configdict["addr"]
         ac1.update_config(configdict)
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
@@ -2459,7 +2557,7 @@ class TestOnlineConfigureFails:
     def test_invalid_domain(self, acfactory):
         configdict = acfactory.get_next_liveconfig()
         ac1 = acfactory.get_unconfigured_account()
-        configdict["addr"] += "$"
+        configdict["addr"] += "x"
         ac1.update_config(configdict)
         configtracker = ac1.configure()
         configtracker.wait_progress(500)

python/tests/test_2_increation.py

@@ -26,7 +26,7 @@ def wait_msgs_changed(account, msgs_list):
                     break
         else:
             account.log(f"waiting mismatch data1={data1} data2={data2}")
-    return ev.data2
+    return ev.data1, ev.data2
 
 
 class TestOnlineInCreation:
@@ -68,17 +68,14 @@ class TestOnlineInCreation:
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
 
-        lp.sec("create a new group")
-        chat2 = ac1.create_group_chat("newgroup")
-        wait_msgs_changed(ac1, [(0, 0)])
-
-        lp.sec("add a contact to new group")
-        chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(chat2.id, None)])
-
         lp.sec("forward the message while still in creation")
+        chat2 = ac1.create_group_chat("newgroup")
+        chat2.add_contact(ac2)
+        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
-        forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        # XXX there might be two EVENT_MSGS_CHANGED and only one of them
+        # is the one caused by forwarding
+        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 

python/tests/test_3_offline.py

@@ -5,6 +5,8 @@ from datetime import datetime, timedelta, timezone
 import pytest
 
 import deltachat as dc
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
 from deltachat.tracker import ImexFailed
 from deltachat import Account, account_hookimpl, Message
 
@@ -800,7 +802,6 @@ class TestOfflineChat:
     def test_audit_log_view_without_daymarker(self, ac1, lp):
         lp.sec("ac1: test audit log (show only system messages)")
         chat = ac1.create_group_chat(name="audit log sample data")
-
         # promote chat
         chat.send_text("hello")
         assert chat.is_promoted()
@@ -810,6 +811,12 @@ class TestOfflineChat:
         chat.set_name("audit log test group")
         chat.send_text("a message in between")
 
+        lp.sec("check message count of all messages")
+        assert len(chat.get_messages()) == 4
+
         lp.sec("check message count of only system messages (without daymarkers)")
-        sysmessages = [x for x in chat.get_messages() if x.is_system_message()]
-        assert len(sysmessages) == 3
+        dc_array = ffi.gc(
+            lib.dc_get_chat_msgs(ac1._dc_context, chat.id, dc.const.DC_GCM_INFO_ONLY, 0),
+            lib.dc_array_unref,
+        )
+        assert len(list(iter_array(dc_array, lambda x: x))) == 2

python/tox.ini

@@ -16,7 +16,7 @@ setenv =
 passenv = 
     DCC_RS_DEV
     DCC_RS_TARGET
-    CHATMAIL_DOMAIN
+    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -62,9 +62,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+# Pinned due to incompatibility of breathe with sphinx 7.2: <https://github.com/breathe-doc/breathe/issues/943>
+    sphinx<=7.1.2
     breathe
-    sphinx_rtd_theme
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 
@@ -85,7 +85,7 @@ commands =
 addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 300
+timeout = 150
 timeout_func_only = True
 markers = 
     ignored: ignore this test in default test runs, use --ignored to run.

release-date.in

@@ -1 +1 @@
-2023-11-21
+2023-10-26

scripts/README.md

@@ -14,23 +14,16 @@ and an own build machine.
 
 - `../.github/workflows` contains jobs run by GitHub Actions.
 
-- `run-python-test.sh` runs CFFI Python tests.
-
-- `run-rpc-test.sh` runs JSON-RPC Python tests.
-
-- `make-python-testenv.sh` creates a local python test development environment with CFFI bindings.
-  Reusing the same environment is faster than running `run-python-test.sh` which always
-  recreates environment from scratch and runs additional lints.
-
-- `make-rpc-testenv.sh` creates a local python development environment with JSON-RPC bindings,
-   i.e. `deltachat-rpc-client` and `deltachat-rpc-server`.
-
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
 
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
+- `make-python-testenv.sh` creates local python test development environment.
+  Reusing the same environment is faster than running `run-python-test.sh` which always
+  recreates environment from scratch and runs additional lints.
+
 - `run-doxygen.sh` generates C-docs which are then uploaded to https://c.delta.chat/
 
 - `run_all.sh` builds Python wheels
@@ -39,8 +32,6 @@ and an own build machine.
 
 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.
 
-- `build-python-docs.sh` builds Python documentation into `dist/html/`.
-
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 

scripts/build-python-docs.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export DCC_RS_TARGET=debug
-export DCC_RS_DEV="$PWD"
-cargo build -p deltachat_ffi --features jsonrpc
-
-python3 -m venv venv
-venv/bin/pip install ./python
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/pip install sphinx breathe sphinx_rtd_theme
-venv/bin/pip install ./deltachat-rpc-client
-venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/concourse/docs_wheels.yml

@@ -102,6 +102,8 @@ jobs:
             - name: deltachat-core-rust-release
               path: .
           outputs:
+            - name: py-docs
+              path: ./python/doc/_build/
             # Binary wheels
             - name: py-wheels
               path: ./python/.docker-tox/wheelhouse/
@@ -113,6 +115,28 @@ jobs:
               - |
                 scripts/run_all.sh
 
+      # Upload python docs to py.delta.chat
+      - task: upload-py-docs
+        config:
+          inputs:
+            - name: py-docs
+          image_resource:
+            type: registry-image
+            source:
+              repository: alpine
+          platform: linux
+          run:
+            path: sh
+            args:
+              - -ec
+              - |
+                apk add --no-cache rsync openssh-client
+                mkdir -p ~/.ssh
+                chmod 700 ~/.ssh
+                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
+                chmod 600 ~/.ssh/id_ed25519
+                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
+
       # Upload x86_64 wheels and source packages
       - task: upload-wheels
         config:

scripts/make-rpc-testenv.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-tox -c deltachat-rpc-client -e py --devenv venv
-venv/bin/pip install --upgrade pip
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug

scripts/remote_tests_python.sh

@@ -28,7 +28,7 @@ ssh $SSHTARGET <<_HERE
     export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
     export TARGET=release
-    export CHATMAIL_DOMAIN=$CHATMAIL_DOMAIN
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

scripts/run-rpc-test.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cargo install --path deltachat-rpc-server/ --root "$PWD/venv" --debug
-PATH="$PWD/venv/bin:$PATH" tox -c deltachat-rpc-client

scripts/run_all.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Build the Delta Chat Core Rust library and Python wheels
+# Build the Delta Chat Core Rust library, Python wheels and docs 
 
 set -e -x
 
@@ -27,10 +27,16 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
-unset CHATMAIL_DOMAIN
+unset DCC_NEW_TMP_EMAIL
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
 tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+tox --workdir "$TOXWORKDIR" -e doc

scripts/update-provider-database.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 export TZ=UTC
 
 # Provider database revision.
-REV=18f714cf73d0bdfb8b013fa344494ab80c92b477
+REV=3c8f7e846c915a183dc44536fb5480d1f25d7c42
 
 CORE_ROOT="$PWD"
 TMP="$(mktemp -d)"

spec.md

@@ -192,7 +192,7 @@ in different orders, esp. on creating new groups.
 To remove a member, a `Chat-Group-Member-Removed` header must be sent
 with the value set to the email-address of the member to remove.
 When receiving a `Chat-Group-Member-Removed` header,
-only exactly the given member has to be removed from the member list.
+only exaxtly the given member has to be removed from the member list.
 
 Messenger clients MUST NOT construct the member list
 on other group messages
@@ -339,7 +339,7 @@ only on image changes.
 
 In older specs, the profile-image was sent as an attachment
 and `Chat-User-Avatar:` specified its name.
-However, it turned out that these attachments are kind of unexpected to users,
+However, it turned out that these attachments are kind of unuexpected to users,
 therefore the profile-image go to the header now.
 
 

src/accounts.rs

@@ -258,8 +258,8 @@ impl Accounts {
     }
 
     /// Starts background tasks such as IMAP and SMTP loops for all accounts.
-    pub async fn start_io(&mut self) {
-        for account in self.accounts.values_mut() {
+    pub async fn start_io(&self) {
+        for account in self.accounts.values() {
             account.start_io().await;
         }
     }

src/aheader.rs

@@ -67,7 +67,7 @@ impl Aheader {
 
 impl fmt::Display for Aheader {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(fmt, "addr={};", self.addr.to_lowercase())?;
+        write!(fmt, "addr={};", self.addr)?;
         if self.prefer_encrypt == EncryptPreference::Mutual {
             write!(fmt, " prefer-encrypt=mutual;")?;
         }
@@ -262,16 +262,5 @@ mod tests {
             )
         )
         .contains("prefer-encrypt"));
-
-        // Always lowercase the address in the header.
-        assert!(format!(
-            "{}",
-            Aheader::new(
-                "TeSt@eXaMpLe.cOm".to_string(),
-                SignedPublicKey::from_base64(RAWKEY).unwrap(),
-                EncryptPreference::Mutual
-            )
-        )
-        .contains("test@example.com"));
     }
 }

src/chatlist.rs

@@ -761,40 +761,4 @@ mod tests {
         let summary = chats.get_summary(&t, 0, None).await.unwrap();
         assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
     }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_load_broken() {
-        let t = TestContext::new_bob().await;
-        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "a chat")
-            .await
-            .unwrap();
-        create_group_chat(&t, ProtectionStatus::Unprotected, "b chat")
-            .await
-            .unwrap();
-        create_group_chat(&t, ProtectionStatus::Unprotected, "c chat")
-            .await
-            .unwrap();
-
-        // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        assert_eq!(chats.len(), 3);
-
-        // obfuscated one chat
-        t.sql
-            .execute("UPDATE chats SET type=10 WHERE id=?", (chat_id1,))
-            .await
-            .unwrap();
-
-        // obfuscated chat can't be loaded
-        assert!(Chat::load_from_db(&t, chat_id1).await.is_err());
-
-        // chatlist loads fine
-        let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-
-        // only corrupted chat fails to create summary
-        assert!(chats.get_summary(&t, 0, None).await.is_ok());
-        assert!(chats.get_summary(&t, 1, None).await.is_ok());
-        assert!(chats.get_summary(&t, 2, None).await.is_err());
-        assert_eq!(chats.get_index_for_id(chat_id1).unwrap(), 2);
-    }
 }

src/config.rs

@@ -266,10 +266,6 @@ pub enum Config {
     /// True if it is a bot account.
     Bot,
 
-    /// True when to skip initial start messages in groups.
-    #[strum(props(default = "0"))]
-    SkipStartMessages,
-
     /// Whether we send a warning if the password is wrong (set to false when we send a warning
     /// because we do not want to send a second warning)
     #[strum(props(default = "0"))]
@@ -302,7 +298,7 @@ pub enum Config {
     DownloadLimit,
 
     /// Enable sending and executing (applying) sync messages. Sending requires `BccSelf` to be set.
-    #[strum(props(default = "1"))]
+    #[strum(props(default = "0"))]
     SyncMsgs,
 
     /// Space-separated list of all the authserv-ids which we believe
@@ -513,11 +509,6 @@ impl Context {
                 );
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
-            Config::Addr => {
-                self.sql
-                    .set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
-                    .await?;
-            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
@@ -667,21 +658,6 @@ mod tests {
         );
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_set_config_addr() {
-        let t = TestContext::new().await;
-
-        // Test that uppercase address get lowercased.
-        assert!(t
-            .set_config(Config::Addr, Some("Foobar@eXample.oRg"))
-            .await
-            .is_ok());
-        assert_eq!(
-            t.get_config(Config::Addr).await.unwrap().unwrap(),
-            "foobar@example.org"
-        );
-    }
-
     /// Tests that "bot" config can only be set to "0" or "1".
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_config_bot() {

src/configure.rs

@@ -31,6 +31,7 @@ use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
 use crate::provider::{Protocol, Socket, UsernamePattern};
+use crate::scheduler::InterruptInfo;
 use crate::smtp::Smtp;
 use crate::socks::Socks5Config;
 use crate::stock_str;
@@ -480,7 +481,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
 
     ctx.set_config_bool(Config::FetchedExistingMsgs, false)
         .await?;
-    ctx.scheduler.interrupt_inbox().await;
+    ctx.scheduler
+        .interrupt_inbox(InterruptInfo::new(false))
+        .await;
 
     progress!(ctx, 940);
     update_device_chats_handle.await??;

src/contact.rs

@@ -19,7 +19,7 @@ use tokio::task;
 use tokio::time::{timeout, Duration};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{ChatId, ProtectionStatus};
+use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY};
@@ -32,7 +32,6 @@ use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::sql::{self, params_iter};
-use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
@@ -43,43 +42,43 @@ use crate::{chat, stock_str};
 const SEEN_RECENTLY_SECONDS: i64 = 600;
 
 /// Valid contact address.
-#[derive(Debug, Clone)]
-pub(crate) struct ContactAddress(String);
+#[derive(Debug, Clone, Copy)]
+pub(crate) struct ContactAddress<'a>(&'a str);
 
-impl Deref for ContactAddress {
+impl Deref for ContactAddress<'_> {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
-        &self.0
+        self.0
     }
 }
 
-impl AsRef<str> for ContactAddress {
+impl AsRef<str> for ContactAddress<'_> {
     fn as_ref(&self) -> &str {
-        &self.0
+        self.0
     }
 }
 
-impl fmt::Display for ContactAddress {
+impl fmt::Display for ContactAddress<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.0)
     }
 }
 
-impl ContactAddress {
+impl<'a> ContactAddress<'a> {
     /// Constructs a new contact address from string,
     /// normalizing and validating it.
-    pub fn new(s: &str) -> Result<Self> {
+    pub fn new(s: &'a str) -> Result<Self> {
         let addr = addr_normalize(s);
-        if !may_be_valid_addr(&addr) {
+        if !may_be_valid_addr(addr) {
             bail!("invalid address {:?}", s);
         }
-        Ok(Self(addr.to_string()))
+        Ok(Self(addr))
     }
 }
 
 /// Allow converting [`ContactAddress`] to an SQLite type.
-impl rusqlite::types::ToSql for ContactAddress {
+impl rusqlite::types::ToSql for ContactAddress<'_> {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         let val = rusqlite::types::Value::Text(self.0.to_string());
         let out = rusqlite::types::ToSqlOutput::Owned(val);
@@ -149,22 +148,6 @@ impl ContactId {
             .await?;
         Ok(())
     }
-
-    /// Reset gossip timestamp in all chats with this contact.
-    pub(crate) async fn regossip_keys(&self, context: &Context) -> Result<()> {
-        context
-            .sql
-            .execute(
-                "UPDATE chats
-                 SET gossiped_timestamp=0
-                 WHERE EXISTS (SELECT 1 FROM chats_contacts
-                               WHERE chats_contacts.chat_id=chats.id
-                               AND chats_contacts.contact_id=?)",
-                (self,),
-            )
-            .await?;
-        Ok(())
-    }
 }
 
 impl fmt::Display for ContactId {
@@ -476,12 +459,12 @@ impl Contact {
 
     /// Block the given contact.
     pub async fn block(context: &Context, id: ContactId) -> Result<()> {
-        set_blocked(context, Sync, id, true).await
+        set_block_contact(context, id, true).await
     }
 
     /// Unblock the given contact.
     pub async fn unblock(context: &Context, id: ContactId) -> Result<()> {
-        set_blocked(context, Sync, id, false).await
+        set_block_contact(context, id, false).await
     }
 
     /// Add a single contact as a result of an _explicit_ user action.
@@ -500,7 +483,7 @@ impl Contact {
         let addr = ContactAddress::new(&addr)?;
 
         let (contact_id, sth_modified) =
-            Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated)
+            Contact::add_or_lookup(context, &name, addr, Origin::ManuallyCreated)
                 .await
                 .context("add_or_lookup")?;
         let blocked = Contact::is_blocked_load(context, contact_id).await?;
@@ -511,7 +494,7 @@ impl Contact {
             }
         }
         if blocked {
-            set_blocked(context, Nosync, contact_id, false).await?;
+            Contact::unblock(context, contact_id).await?;
         }
 
         Ok(contact_id)
@@ -540,24 +523,10 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `may_be_valid_addr()`.
-    ///
-    /// Returns the contact ID of the contact belonging to the e-mail address or 0 if there is no
-    /// contact that is or was introduced by an accepted contact.
     pub async fn lookup_id_by_addr(
         context: &Context,
         addr: &str,
         min_origin: Origin,
-    ) -> Result<Option<ContactId>> {
-        Self::lookup_id_by_addr_ex(context, addr, min_origin, Some(Blocked::Not)).await
-    }
-
-    /// The same as `lookup_id_by_addr()`, but internal function. Currently also allows looking up
-    /// not unblocked contacts.
-    pub(crate) async fn lookup_id_by_addr_ex(
-        context: &Context,
-        addr: &str,
-        min_origin: Origin,
-        blocked: Option<Blocked>,
     ) -> Result<Option<ContactId>> {
         if addr.is_empty() {
             bail!("lookup_id_by_addr: empty address");
@@ -565,7 +534,7 @@ impl Contact {
 
         let addr_normalized = addr_normalize(addr);
 
-        if context.is_self_addr(&addr_normalized).await? {
+        if context.is_self_addr(addr_normalized).await? {
             return Ok(Some(ContactId::SELF));
         }
 
@@ -574,14 +543,8 @@ impl Contact {
             .query_get_value(
                 "SELECT id FROM contacts \
             WHERE addr=?1 COLLATE NOCASE \
-            AND id>?2 AND origin>=?3 AND (? OR blocked=?)",
-                (
-                    &addr_normalized,
-                    ContactId::LAST_SPECIAL,
-                    min_origin as u32,
-                    blocked.is_none(),
-                    blocked.unwrap_or_default(),
-                ),
+            AND id>?2 AND origin>=?3 AND blocked=0;",
+                (&addr_normalized, ContactId::LAST_SPECIAL, min_origin as u32),
             )
             .await?;
         Ok(id)
@@ -615,7 +578,7 @@ impl Contact {
     pub(crate) async fn add_or_lookup(
         context: &Context,
         name: &str,
-        addr: &ContactAddress,
+        addr: ContactAddress<'_>,
         mut origin: Origin,
     ) -> Result<(ContactId, Modifier)> {
         let mut sth_modified = Modifier::None;
@@ -623,7 +586,7 @@ impl Contact {
         ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
-        if context.is_self_addr(addr).await? {
+        if context.is_self_addr(&addr).await? {
             return Ok((ContactId::SELF, sth_modified));
         }
 
@@ -778,7 +741,7 @@ impl Contact {
                             } else {
                                 "".to_string()
                             },
-                            &addr,
+                            addr,
                             origin,
                             if update_authname {
                                 name.to_string()
@@ -823,7 +786,7 @@ impl Contact {
             let name = normalize_name(&name);
             match ContactAddress::new(&addr) {
                 Ok(addr) => {
-                    match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
+                    match Contact::add_or_lookup(context, &name, addr, Origin::AddressBook).await {
                         Ok((_, modified)) => {
                             if modified != Modifier::None {
                                 modify_cnt += 1
@@ -1267,20 +1230,10 @@ impl Contact {
         self.status.as_str()
     }
 
-    /// Returns true if the contact
-    /// can be added to verified chats,
-    /// i.e. has a verified key
-    /// and Autocrypt key matches the verified key.
+    /// Check if a contact was verified. E.g. by a secure-join QR code scan
+    /// and if the key has not changed since this verification.
     ///
-    /// If contact is verified
-    /// UI should display green checkmark after the contact name
-    /// in contact list items and
-    /// in chat member list items.
-    ///
-    /// In contact profile view, us this function only if there is no chat with the contact,
-    /// otherwise use is_chat_protected().
-    /// Use [Self::get_verifier_id] to display the verifier contact
-    /// in the info section of the contact profile.
+    /// The UI may draw a checkbox or something like that beside verified contacts.
     pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
@@ -1297,27 +1250,20 @@ impl Contact {
         Ok(VerifiedStatus::Unverified)
     }
 
-    /// Returns the `ContactId` that verified the contact.
-    ///
-    /// If the function returns non-zero result,
-    /// display green checkmark in the profile and "Introduced by ..." line
-    /// with the name and address of the contact
-    /// formatted by [Self::get_name_n_addr].
-    ///
-    /// If this function returns a verifier,
-    /// this does not necessarily mean
-    /// you can add the contact to verified chats.
-    /// Use [Self::is_verified] to check
-    /// if a contact can be added to a verified chat instead.
-    pub async fn get_verifier_id(&self, context: &Context) -> Result<Option<ContactId>> {
-        let Some(verifier_addr) = Peerstate::from_addr(context, self.get_addr())
+    /// Returns the address that verified the contact.
+    pub async fn get_verifier_addr(&self, context: &Context) -> Result<Option<String>> {
+        Ok(Peerstate::from_addr(context, self.get_addr())
             .await?
-            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned()))
-        else {
+            .and_then(|peerstate| peerstate.get_verifier().map(|addr| addr.to_owned())))
+    }
+
+    /// Returns the ContactId that verified the contact.
+    pub async fn get_verifier_id(&self, context: &Context) -> Result<Option<ContactId>> {
+        let Some(verifier_addr) = self.get_verifier_addr(context).await? else {
             return Ok(None);
         };
 
-        if addr_cmp(&verifier_addr, &self.addr) {
+        if verifier_addr == self.addr {
             // Contact is directly verified via QR code.
             return Ok(Some(ContactId::SELF));
         }
@@ -1332,27 +1278,6 @@ impl Contact {
         }
     }
 
-    /// Returns if the contact profile title should display a green checkmark.
-    ///
-    /// This generally should be consistent with the 1:1 chat with the contact
-    /// so 1:1 chat with the contact and the contact profile
-    /// either both display the green checkmark or both don't display a green checkmark.
-    ///
-    /// UI often knows beforehand if a chat exists and can also call
-    /// `chat.is_protected()` (if there is a chat)
-    /// or `contact.is_verified()` (if there is no chat) directly.
-    /// This is often easier and also skips some database calls.
-    pub async fn is_profile_verified(&self, context: &Context) -> Result<bool> {
-        let contact_id = self.id;
-
-        if let Some(chat_id) = ChatId::lookup_by_contact(context, contact_id).await? {
-            Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
-        } else {
-            // 1:1 chat does not exist.
-            Ok(self.is_verified(context).await? == VerifiedStatus::BidirectVerified)
-        }
-    }
-
     /// Returns the number of real (i.e. non-special) contacts in the database.
     pub async fn get_real_cnt(context: &Context) -> Result<usize> {
         if !context.sql.is_open().await {
@@ -1405,13 +1330,12 @@ pub fn may_be_valid_addr(addr: &str) -> bool {
     res.is_ok()
 }
 
-/// Returns address lowercased,
-/// with whitespace trimmed and `mailto:` prefix removed.
-pub fn addr_normalize(addr: &str) -> String {
-    let norm = addr.trim().to_lowercase();
+/// Returns address with whitespace trimmed and `mailto:` prefix removed.
+pub fn addr_normalize(addr: &str) -> &str {
+    let norm = addr.trim();
 
     if norm.starts_with("mailto:") {
-        norm.get(7..).unwrap_or(&norm).to_string()
+        norm.get(7..).unwrap_or(norm)
     } else {
         norm
     }
@@ -1439,9 +1363,8 @@ fn sanitize_name_and_addr(name: &str, addr: &str) -> (String, String) {
     }
 }
 
-pub(crate) async fn set_blocked(
+async fn set_block_contact(
     context: &Context,
-    sync: sync::Sync,
     contact_id: ContactId,
     new_blocking: bool,
 ) -> Result<()> {
@@ -1450,6 +1373,7 @@ pub(crate) async fn set_blocked(
         "Can't block special contact {}",
         contact_id
     );
+
     let contact = Contact::get_by_id(context, contact_id).await?;
 
     if contact.blocked != new_blocking {
@@ -1491,23 +1415,9 @@ WHERE type=? AND id IN (
             if let Some((chat_id, _, _)) =
                 chat::get_chat_id_by_grpid(context, &contact.addr).await?
             {
-                chat_id.unblock_ex(context, Nosync).await?;
+                chat_id.unblock(context).await?;
             }
         }
-
-        if sync.into() {
-            let action = match new_blocking {
-                true => chat::SyncAction::Block,
-                false => chat::SyncAction::Unblock,
-            };
-            context
-                .add_sync_item(SyncData::AlterChat {
-                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
-                    action,
-                })
-                .await?;
-            context.send_sync_msg().await?;
-        }
     }
 
     Ok(())
@@ -1666,8 +1576,8 @@ fn cat_fingerprint(
 
 /// Compares two email addresses, normalizing them beforehand.
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
-    let norm1 = addr_normalize(addr1);
-    let norm2 = addr_normalize(addr2);
+    let norm1 = addr_normalize(addr1).to_lowercase();
+    let norm2 = addr_normalize(addr2).to_lowercase();
 
     norm1 == norm2
 }
@@ -1865,7 +1775,10 @@ mod tests {
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");
         assert_eq!(addr_normalize("  hello@world.com   "), "hello@world.com");
-        assert_eq!(addr_normalize("John@Doe.com"), "john@doe.com");
+
+        // normalisation preserves case to allow user-defined spelling.
+        // however, case is ignored on addr_cmp()
+        assert_ne!(addr_normalize("John@Doe.com"), "john@doe.com");
     }
 
     #[test]
@@ -1891,7 +1804,7 @@ mod tests {
         let (id, _modified) = Contact::add_or_lookup(
             &context.ctx,
             "bob",
-            &ContactAddress::new("user@example.org")?,
+            ContactAddress::new("user@example.org")?,
             Origin::IncomingReplyTo,
         )
         .await?;
@@ -1919,7 +1832,7 @@ mod tests {
         let (contact_bob_id, modified) = Contact::add_or_lookup(
             &context.ctx,
             "someone",
-            &ContactAddress::new("user@example.org")?,
+            ContactAddress::new("user@example.org")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -1984,7 +1897,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bla foo",
-            &ContactAddress::new("one@eins.org").unwrap(),
+            ContactAddress::new("one@eins.org").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2003,7 +1916,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "Real one",
-            &ContactAddress::new(" one@eins.org  ").unwrap(),
+            ContactAddress::new(" one@eins.org  ").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -2019,7 +1932,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            &ContactAddress::new("three@drei.sam").unwrap(),
+            ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2036,7 +1949,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "m. serious",
-            &ContactAddress::new("three@drei.sam").unwrap(),
+            ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2051,7 +1964,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "schnucki",
-            &ContactAddress::new("three@drei.sam").unwrap(),
+            ContactAddress::new("three@drei.sam").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -2067,7 +1980,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            &ContactAddress::new("alice@w.de").unwrap(),
+            ContactAddress::new("alice@w.de").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2209,7 +2122,7 @@ mod tests {
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            &ContactAddress::new("bob@example.net")?,
+            ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2288,7 +2201,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob1",
-            &ContactAddress::new("bob@example.org").unwrap(),
+            ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2304,7 +2217,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob2",
-            &ContactAddress::new("bob@example.org").unwrap(),
+            ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2330,7 +2243,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob4",
-            &ContactAddress::new("bob@example.org").unwrap(),
+            ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2359,7 +2272,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire1",
-            &ContactAddress::new("claire@example.org").unwrap(),
+            ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2375,7 +2288,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire2",
-            &ContactAddress::new("claire@example.org").unwrap(),
+            ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2400,7 +2313,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            &ContactAddress::new("bob@example.org")?,
+            ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2412,7 +2325,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Not Bob",
-            &ContactAddress::new("bob@example.org")?,
+            ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownTo,
         )
         .await?;
@@ -2425,7 +2338,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            &ContactAddress::new("bob@example.org")?,
+            ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2454,7 +2367,7 @@ mod tests {
         Contact::add_or_lookup(
             &t,
             "dave2",
-            &ContactAddress::new("dave@example.org").unwrap(),
+            ContactAddress::new("dave@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2575,7 +2488,7 @@ mod tests {
         let (contact_bob_id, _modified) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            &ContactAddress::new("bob@example.net")?,
+            ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2738,7 +2651,7 @@ CCCB 5AA9 F6E1 141C 9431
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            &ContactAddress::new("bob@example.net")?,
+            ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2797,6 +2710,7 @@ Hi."#;
 
         let contact_id = Contact::create(&alice, "Bob", "bob@example.net").await?;
         let contact = Contact::get_by_id(&alice, contact_id).await?;
+        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         // Receive a message from Bob to create a peerstate.
@@ -2805,6 +2719,7 @@ Hi."#;
         alice.recv_msg(&sent_msg).await;
 
         let contact = Contact::get_by_id(&alice, contact_id).await?;
+        assert!(contact.get_verifier_addr(&alice).await?.is_none());
         assert!(contact.get_verifier_id(&alice).await?.is_none());
 
         Ok(())