add integration test

webxdcs can now have multiple gossip channels and decide where to send the message to.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
     name: Lint Rust
     runs-on: ubuntu-latest
     env:
-      RUSTUP_TOOLCHAIN: 1.73.0
+      RUSTUP_TOOLCHAIN: 1.75.0
     steps:
       - uses: actions/checkout@v3
       - name: Install rustfmt and clippy
@@ -76,15 +76,15 @@ jobs:
       matrix:
         include:
           - os: ubuntu-latest
-            rust: 1.73.0
+            rust: 1.75.0
           - os: windows-latest
-            rust: 1.73.0
+            rust: 1.75.0
           - os: macos-latest
-            rust: 1.73.0
+            rust: 1.75.0
 
-          # Minimum Supported Rust Version = 1.70.0
+          # Minimum Supported Rust Version = 1.72.0
           - os: ubuntu-latest
-            rust: 1.70.0
+            rust: 1.72.0
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

.github/workflows/upload-python-docs.yml

@@ -0,0 +1,25 @@
+name: Build & Deploy Documentation on py.delta.chat
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # Fetch history to calculate VCS version number.
+      - name: Build Python documentation
+        run: scripts/build-python-docs.sh
+      - name: Upload to py.delta.chat
+        uses: up9cloud/action-rsync@v1.3
+        env:
+          USER: delta
+          KEY: ${{ secrets.CODESPEAK_KEY }}
+          HOST: "lists.codespeak.net"
+          SOURCE: "dist/html/"
+          TARGET: "/home/delta/build/master"

CHANGELOG.md

@@ -1,5 +1,232 @@
 # Changelog
 
+## [1.133.1] - 2024-01-21
+
+### API-Changes
+
+- Add `is_bot` to cffi and jsonrpc ([#5197](https://github.com/deltachat/deltachat-core-rust/pull/5197)).
+
+### Features / Changes
+
+- Add system message when provider does not allow unencrypted messages ([#5195](https://github.com/deltachat/deltachat-core-rust/pull/5195)).
+
+### Fixes
+
+- `Chat::send_msg`: Remove encryption-related params from already sent message. This allows to send received encrypted `dc_msg_t` object to unencrypted chat, e.g. in a Python bot.
+- Set message download state to Failure on IMAP errors. This avoids partially downloaded messages getting stuck in "Downloading..." state without actually being in a download queue.
+- BCC-to-self even if server deletion is set to "at once". This is a workaround for SMTP servers which do not return response in time, BCC-self works as a confirmation that message was sent out successfully and does not need more retries.
+- node: Run tests with native ESM modules instead of `esm` ([#5194](https://github.com/deltachat/deltachat-core-rust/pull/5194)).
+- Use Quoted-Printable MIME encoding for the text part ([#3986](https://github.com/deltachat/deltachat-core-rust/pull/3986)).
+
+### Tests
+
+- python: Add `get_protected_chat` to testplugin.py.
+
+## [1.133.0] - 2024-01-14
+
+### Features / Changes
+
+- Securejoin protocol implementation refinements
+  - Track forward and backward verification separately ([#5089](https://github.com/deltachat/deltachat-core-rust/pull/5089)) to avoid inconsistent states.
+  - Mark 1:1 chat as verified for Bob early. 1:1 chat with Alice is verified as soon as Alice's key is verified rather than at the end of the protocol.
+- Put Message-ID into hidden headers and take it from there on receiver ([#4798](https://github.com/deltachat/deltachat-core-rust/pull/4798)). This works around servers which generate their own Message-ID and overwrite the one generated by Delta Chat.
+- deltachat-repl: Enable INFO logging by default and add timestamps.
+- Add `ConfigSynced` (`DC_EVENT_CONFIG_SYNCED`) event which is emitted when configuration is changed via synchronization message or synchronization message for configuration is sent. UI may refresh elments based on the configuration key which is a part of the event.
+- Sync contact creation/rename across devices ([#5163](https://github.com/deltachat/deltachat-core-rust/pull/5163)).
+- Encrypt MDNs ([#5175](https://github.com/deltachat/deltachat-core-rust/pull/5175)).
+- Only try to configure non-strict TLS checks if explicitly set ([#5181](https://github.com/deltachat/deltachat-core-rust/pull/5181)).
+
+### Build system
+
+- Use released version of iroh 0.4.2 for "setup second device" feature.
+
+### CI
+
+- Update to Rust 1.75.0.
+- Downgrade `chai` from 4.4.0 to 4.3.10.
+
+### Documentation
+
+- Add a link <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html> to autoconfig RFC draft.
+- Update securejoin link in `standards.md` from <https://countermitm.readthedocs.io/> to <https://securejoin.readthedocs.io>.
+- Restore "Constants" page in Doxygen >=1.9.8
+
+### Fixes
+
+- imap: Limit the rate of LOGIN attempts rather than connection attempts. This is to avoid having to wait for rate limiter right after switching from a bad or offline network to a working network while still guarding against reconnection loop.
+- Do not ignore `peerstate.save_to_db()` errors.
+- securejoin: Mark 1:1s as protected regardless of the Config::VerifiedOneOnOneChats.
+- Delete received outgoing messages from SMTP queue ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- imap: Fail fast on `LIST` errors to avoid busy loop when connection is lost.
+- Split SMTP jobs already in `chat::create_send_msg_jobs()` ([#5115](https://github.com/deltachat/deltachat-core-rust/pull/5115)).
+- Do not remove contents from unencrypted [Schleuder](https://schleuder.org/) mailing lists messages.
+- Reset message error when scheduling resending ([#5119](https://github.com/deltachat/deltachat-core-rust/pull/5119)).
+- Emit events more reliably when starting and stopping I/O ([#5101](https://github.com/deltachat/deltachat-core-rust/pull/5101)).
+- Fix timestamp of chat protection info message for correct message ordering after restoring a backup ([#5088](https://github.com/deltachat/deltachat-core-rust/pull/5088)).
+
+### Refactor
+
+- sql: Recreate `config` table with UNIQUE constraint.
+- sql: Recreate `keypairs` table to remove unused `addr` and `created` fields and move `is_default` flag to `config` table.
+- Send `Secure-Join-Fingerprint` only in `*-request-with-auth`.
+
+### Tests
+
+- Test joining non-protected group.
+- Test that read receipts don't degrade encryption.
+- Test that changing default private key breaks backward verification.
+- Test recovery from lost vc-contact-confirm.
+- Use `wait_for_incoming_msg_event()` more.
+
+## [1.132.1] - 2023-12-12
+
+### Features / Changes
+
+- Add "From:" to protected headers for signed-only messages.
+- Sync user actions for ad-hoc groups across devices ([#5065](https://github.com/deltachat/deltachat-core-rust/pull/5065)).
+
+### Fixes
+
+- Add padlock to empty part if the whole message is empty.
+- Renew IDLE timeout on keepalives and reduce it to 5 minutes.
+- connectivity: Return false from `all_work_done()` immediately after connecting (iOS notification fix).
+
+### API-Changes
+
+- deltachat-jsonrpc-client: add `Account.{import,export}_self_keys`.
+
+### CI
+
+- Update to Rust 1.74.1.
+
+## [1.132.0] - 2023-12-06
+
+### Features / Changes
+
+- Increase TCP timeouts from 30 to 60 seconds.
+
+### Fixes
+
+- Don't sort message creating a protected group over a protection message ([#4963](https://github.com/deltachat/deltachat-core-rust/pull/4963)).
+- Do not lock accounts.toml on iOS.
+- Protect groups even if some members are not verified and add `test_securejoin_after_contact_resetup` regression test.
+
+## [1.131.9] - 2023-12-02
+
+### API-Changes
+
+- Remove `dc_get_http_response()`, `dc_http_response_get_mimetype()`, `dc_http_response_get_encoding()`, `dc_http_response_get_blob()`, `dc_http_response_get_size()`, `dc_http_response_unref()` and `dc_http_response_t` from cffi.
+- Deprecate CFFI APIs `dc_send_reaction()`, `dc_get_msg_reactions()`, `dc_reactions_get_contacts()`, `dc_reactions_get_by_contact_id()`, `dc_reactions_unref` and `dc_reactions_t`.
+- Make `Contact.is_verified()` return bool.
+
+### Build system
+
+- Switch from fork of iroh to iroh 0.4.2 pre-release.
+
+### Features / Changes
+
+- Send `Chat-Verified` headers in 1:1 chats.
+- Ratelimit IMAP connections ([#4940](https://github.com/deltachat/deltachat-core-rust/pull/4940)).
+- Remove receiver limit on `.xdc` size.
+- Don't affect MimeMessage with "From" and secured headers from encrypted unsigned messages.
+- Sync `Config::{MdnsEnabled,ShowEmails}` across devices ([#4954](https://github.com/deltachat/deltachat-core-rust/pull/4954)).
+- Sync `Config::Displayname` across devices ([#4893](https://github.com/deltachat/deltachat-core-rust/pull/4893)).
+- `Chat::rename_ex`: Don't send sync message if usual message is sent.
+
+### Fixes
+
+- Lock the database when INSERTing a webxdc update, avoid "Database is locked" errors.
+- Use keyring with all private keys when decrypting a message ([#5046](https://github.com/deltachat/deltachat-core-rust/pull/5046)).
+
+### Tests
+
+- Make Result-returning tests produce a line number.
+- Add `test_utils::sync()`.
+- Test inserting lots of webxdc updates.
+- Split `test_sync_alter_chat()` into smaller tests.
+
+## [1.131.8] - 2023-11-27
+
+### Features / Changes
+
+- webxdc: Add unique IDs to status updates sent outside and deduplicate based on IDs.
+
+### Fixes
+
+- Allow IMAP servers not returning UIDNEXT on SELECT and STATUS such as mail.163.com.
+- Use the correct securejoin strings used in the UI, remove old TODO ([#5047](https://github.com/deltachat/deltachat-core-rust/pull/5047)).
+- Do not emit events about webxdc update events logged into debug log webxdc.
+
+### Tests
+
+- Check that `receive_status_update` has forward compatibility and unique webxdc IDs will be ignored by previous Delta Chat versions.
+
+## [1.131.7] - 2023-11-24
+
+### Fixes
+
+- Revert "fix: check UIDNEXT with a STATUS command before going IDLE". This attempts to fix mail.163.com which has broken STATUS command.
+
+## [1.131.6] - 2023-11-21
+
+### Fixes
+
+- Fail fast if IMAP FETCH cannot be parsed instead of getting stuck in infinite loop.
+
+### Documentation
+
+- Generate deltachat-rpc-client documentation and publish it to <https://py.delta.chat>.
+
+## [1.131.5] - 2023-11-20
+
+### API-Changes
+
+- deltachat-rpc-client: Add `Message.get_sender_contact()`.
+- Turn `ContactAddress` into an owned type.
+
+### Features / Changes
+
+- Lowercase addresses in Autocrypt and Autocrypt-Gossip headers.
+- Lowercase the address in member added/removed messages.
+- Lowercase `addr` when it is set.
+- Do not replace the message with an error in square brackets when the sender is not a member of the protected group.
+
+### Fixes
+
+- `Chat::sync_contacts()`: Fetch contact addresses in a single query.
+- `Chat::rename_ex()`: Sync improved chat name to other devices.
+- Recognize `Chat-Group-Member-Added` of self case-insensitively.
+- Compare verifier addr to peerstate addr case-insensitively.
+
+### Tests
+
+- Port [Secure-Join](https://securejoin.readthedocs.io/) tests to JSON-RPC.
+
+### CI
+
+- Test with Rust 1.74.
+
+
+## [1.131.4] - 2023-11-16
+
+### Documentation
+
+- Document DC_DOWNLOAD_UNDECIPHERABLE.
+
+### Fixes
+
+- Always add "Member added" as system message.
+
+## [1.131.3] - 2023-11-15
+
+### Fixes
+
+- Update async-imap to 0.9.4 which does not ignore EOF on FETCH.
+- Reset gossiped timestamp on securejoin.
+- sync: Ignore unknown sync items to provide forward compatibility and avoid creating empty message bubbles.
+- sync: Skip sync when chat name is set to the current one.
+- Return connectivity HTML with an error when IO is stopped.
+
 ## [1.131.2] - 2023-11-14
 
 ### API-Changes
@@ -3199,3 +3426,14 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.131.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.130.0...v1.131.0
 [1.131.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.0...v1.131.1
 [1.131.2]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.1...v1.131.2
+[1.131.3]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.2...v1.131.3
+[1.131.4]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.3...v1.131.4
+[1.131.5]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.4...v1.131.5
+[1.131.6]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.5...v1.131.6
+[1.131.7]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.6...v1.131.7
+[1.131.8]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.7...v1.131.8
+[1.131.9]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.8...v1.131.9
+[1.132.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.131.9...v1.132.0
+[1.132.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.0...v1.132.1
+[1.133.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.132.1...v1.133.0
+[1.133.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.133.0...v1.133.1

CONTRIBUTING.md

@@ -86,6 +86,17 @@ For example:
 .with_context(|| format!("Unable to trash message {msg_id}"))
 ```
 
+All errors should be handled in one of these ways:
+- With `if let Err() =` (incl. logging them into `warn!()`/`err!()`).
+- With `.log_err().ok()`.
+- Bubbled up with `?`.
+
+`backtrace` feature is enabled for `anyhow` crate
+and `debug = 1` option is set in the test profile.
+This allows to run `RUST_BACKTRACE=1 cargo test`
+and get a backtrace with line numbers in resultified tests
+which return `anyhow::Result`.
+
 ### Logging
 
 For logging, use `info!`, `warn!` and `error!` macros.

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "deltachat"
-version = "1.131.2"
+version = "1.133.1"
 edition = "2021"
 license = "MPL-2.0"
-rust-version = "1.70"
+rust-version = "1.72"
 
 [profile.dev]
 debug = 0
@@ -11,6 +11,10 @@ panic = 'abort'
 opt-level = 1
 
 [profile.test]
+# Make anyhow `backtrace` feature useful.
+# With `debug = 0` there are no line numbers in the backtrace
+# produced with RUST_BACKTRACE=1.
+debug = 1
 opt-level = 0
 
 # Always optimize dependencies.
@@ -26,6 +30,10 @@ opt-level = "z"
 codegen-units = 1
 strip = true
 
+[patch.crates-io]
+imap-proto = { git = "https://github.com/djc/tokio-imap.git", rev = "01ff256a7e42a9f7d2732706f8b71a16ce93427e" }
+"iroh-blake3" = { git = "https://github.com/n0-computer/iroh-blake3.git", branch = "master" }
+
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 format-flowed = { path = "./format-flowed" }
@@ -33,26 +41,52 @@ ratelimit = { path = "./deltachat-ratelimit" }
 
 anyhow = "1"
 async-channel = "2.0.0"
-async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] }
-async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] }
-async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] }
-async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] }
+async-imap = { version = "0.9.5", default-features = false, features = [
+  "runtime-tokio",
+] }
+async-native-tls = { version = "0.5", default-features = false, features = [
+  "runtime-tokio",
+] }
+async-smtp = { version = "0.9", default-features = false, features = [
+  "runtime-tokio",
+] }
+async_zip = { version = "0.0.12", default-features = false, features = [
+  "deflate",
+  "fs",
+] }
 backtrace = "0.3"
 base64 = "0.21"
-brotli = { version = "3.4", default-features=false, features = ["std"] }
-chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
+brotli = { version = "3.4", default-features = false, features = ["std"] }
+chrono = { version = "0.4", default-features = false, features = [
+  "clock",
+  "std",
+] }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
-fast-socks5 = "0.8"
-fd-lock = "3.0.11"
+fast-socks5 = "0.9"
+fd-lock = "4"
 futures = "0.3"
 futures-lite = "2.0.0"
 hex = "0.4.0"
 hickory-resolver = "0.24"
 humansize = "2"
-image = { version = "0.24.7", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.24.7", default-features = false, features = [
+  "gif",
+  "jpeg",
+  "ico",
+  "png",
+  "pnm",
+  "webp",
+  "bmp",
+] }
 iroh = { git = "https://github.com/deltachat/iroh", branch = "0.4-update-quic", default-features = false }
+iroh-net = { git = "https://github.com/n0-computer/iroh", branch = "main" }
+iroh-base = { git = "https://github.com/n0-computer/iroh", branch = "main" }
+iroh-gossip = { git = "https://github.com/n0-computer/iroh", branch = "main", features = [
+  "net",
+] }
+quinn = "0.10"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -69,13 +103,14 @@ pin-project = "1"
 pretty_env_logger = { version = "0.5", optional = true }
 qrcodegen = "1.7.0"
 quick-xml = "0.31"
+quoted_printable = "0.5"
 rand = "0.8"
 regex = "1.9"
-reqwest = { version = "0.11.20", features = ["json"] }
+reqwest = { version = "0.11.23", features = ["json"] }
 rusqlite = { version = "0.30", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 sanitize-filename = "0.5"
-serde_json = "1.0"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
 sha2 = "0.10"
@@ -85,7 +120,12 @@ strum_macros = "0.25"
 tagger = "4.3.4"
 textwrap = "0.16.0"
 thiserror = "1"
-tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
+tokio = { version = "1", features = [
+  "fs",
+  "rt-multi-thread",
+  "macros",
+  "time",
+] }
 tokio-io-timeout = "1.2.0"
 tokio-stream = { version = "0.1.14", features = ["fs"] }
 tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
@@ -96,14 +136,21 @@ uuid = { version = "1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 ansi_term = "0.12.0"
+anyhow = { version = "1", features = [
+  "backtrace",
+] } # Enable `backtrace` feature in tests.
 criterion = { version = "0.5.1", features = ["async_tokio"] }
 futures-lite = "2.0.0"
 log = "0.4"
 pretty_env_logger = "0.5"
 proptest = { version = "1", default-features = false, features = ["std"] }
 tempfile = "3"
-testdir = "0.8.0"
-tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"] }
+testdir = "0.9.0"
+tokio = { version = "1", features = [
+  "parking_lot",
+  "rt-multi-thread",
+  "macros",
+] }
 pretty_assertions = "1.3.0"
 
 [workspace]
@@ -151,5 +198,5 @@ internals = []
 vendored = [
   "async-native-tls/vendored",
   "rusqlite/bundled-sqlcipher-vendored-openssl",
-  "reqwest/native-tls-vendored"
+  "reqwest/native-tls-vendored",
 ]

README.md

@@ -27,7 +27,7 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=deltachat_repl=info cargo run -p deltachat-repl -- ~/deltachat-db
+$ cargo run -p deltachat-repl -- ~/deltachat-db
 ```
 where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
@@ -121,7 +121,7 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-- `RUST_LOG=deltachat_repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+- `RUST_LOG=async_imap=trace,async_smtp=trace`: enable IMAP and
 SMTP tracing in addition to info messages.
 
 ### Expensive tests

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.131.2"
+version = "1.133.1"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/DoxygenLayout.xml

@@ -9,7 +9,7 @@
       <tab type="hierarchy" visible="no" title="" intro=""/>
       <tab type="classmembers" visible="no" title="" intro=""/>
     </tab>
-    <tab type="modules" visible="yes" title="Constants" intro="Here is a list of constants:"/>
+    <tab type="topics" visible="yes" title="Constants" intro="Here is a list of constants:"/>
     <tab type="pages" visible="yes" title="" intro=""/>
     <tab type="namespaces" visible="yes" title="">
       <tab type="namespacelist" visible="yes" title="" intro=""/>

deltachat-ffi/deltachat.h

@@ -25,7 +25,6 @@ typedef struct _dc_event     dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 typedef struct _dc_backup_provider dc_backup_provider_t;
-typedef struct _dc_http_response dc_http_response_t;
 
 // Alias for backwards compatibility, use dc_event_emitter_t instead.
 typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
@@ -1112,6 +1111,7 @@ uint32_t dc_send_videochat_invitation (dc_context_t* context, uint32_t chat_id);
  * received overrides all previously received reactions. It is
  * possible to remove all reactions by sending an empty string.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `send_reaction` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id ID of the message you react to.
@@ -1124,6 +1124,7 @@ uint32_t dc_send_reaction (dc_context_t* context, uint32_t msg_id, char *reactio
 /**
  * Get a structure with reactions to the message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_id The message ID to get reactions for.
@@ -2561,7 +2562,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for protected groups as well as for normal groups.
  *     If set to 0, the Setup-Contact protocol is offered in the QR code.
- *     See https://countermitm.readthedocs.io/en/latest/new.html
+ *     See https://securejoin.readthedocs.io/en/latest/new.html
  *     for details about both protocols.
  * @return The text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
@@ -2597,7 +2598,7 @@ char*           dc_get_securejoin_qr_svg         (dc_context_t* context, uint32_
  *
  * Subsequent calls of dc_join_securejoin() will abort previous, unfinished handshakes.
  *
- * See https://countermitm.readthedocs.io/en/latest/new.html
+ * See https://securejoin.readthedocs.io/en/latest/new.html
  * for details about both protocols.
  *
  * @memberof dc_context_t
@@ -4396,6 +4397,9 @@ int             dc_msg_is_info                (const dc_msg_t* msg);
  * Currently, the following types are defined:
  * - DC_INFO_PROTECTION_ENABLED (11) - Info-message for "Chat is now protected"
  * - DC_INFO_PROTECTION_DISABLED (12) - Info-message for "Chat is no longer protected"
+ * - DC_INFO_INVALID_UNENCRYPTED_MAIL (13) - Info-message for "Provider requires end-to-end encryption which is not setup yet",
+ *   the UI should change the corresponding string using #DC_STR_INVALID_UNENCRYPTED_MAIL
+ *   and also offer a way to fix the encryption, eg. by a button offering a QR scan
  *
  * Even when you display an icon,
  * you should still display the text of the informational message using dc_msg_get_text()
@@ -4422,6 +4426,7 @@ int             dc_msg_get_info_type          (const dc_msg_t* msg);
 #define         DC_INFO_EPHEMERAL_TIMER_CHANGED   10
 #define         DC_INFO_PROTECTION_ENABLED        11
 #define         DC_INFO_PROTECTION_DISABLED       12
+#define         DC_INFO_INVALID_UNENCRYPTED_MAIL  13
 #define         DC_INFO_WEBXDC_INFO_MESSAGE       32
 
 /**
@@ -4580,15 +4585,18 @@ int dc_msg_has_html (dc_msg_t* msg);
   * if they are larger than the limit set by the dc_set_config()-option `download_limit`.
   *
   * The function returns one of:
-  * - @ref DC_DOWNLOAD_DONE        - The message does not need any further download action
-  *                                  and should be rendered as usual.
-  * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
-  *                                  In addition to the usual message rendering,
-  *                                  the UI shall show a download button that calls dc_download_full_msg()
-  * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_download_full_msg() and is still in progress.
-  *                                  If the download fails or succeeds,
-  *                                  the event @ref DC_EVENT_MSGS_CHANGED is emitted.
-  * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_download_full_msg() again.
+  * - @ref DC_DOWNLOAD_DONE           - The message does not need any further download action
+  *                                     and should be rendered as usual.
+  * - @ref DC_DOWNLOAD_AVAILABLE      - There is additional content to download.
+  *                                     In addition to the usual message rendering,
+  *                                     the UI shall show a download button that calls dc_download_full_msg()
+  * - @ref DC_DOWNLOAD_IN_PROGRESS    - Download was started with dc_download_full_msg() and is still in progress.
+  *                                     If the download fails or succeeds,
+  *                                     the event @ref DC_EVENT_MSGS_CHANGED is emitted.
+  *
+  * - @ref DC_DOWNLOAD_UNDECIPHERABLE - The message does not need any futher download action.
+  *                                     It was fully downloaded, but we failed to decrypt it.
+  * - @ref DC_DOWNLOAD_FAILURE        - Download error, the user may start over calling dc_download_full_msg() again.
   *
   * @memberof dc_msg_t
   * @param msg The message object.
@@ -5064,6 +5072,15 @@ int             dc_contact_is_blocked        (const dc_contact_t* contact);
  */
 int             dc_contact_is_verified       (dc_contact_t* contact);
 
+/**
+ * Returns whether contact is a bot.
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 0 if the contact is not a bot, 1 otherwise.
+ */
+int             dc_contact_is_bot            (dc_contact_t* contact);
+
 
 /**
  * Return the contact ID that verified a contact.
@@ -5182,72 +5199,6 @@ int             dc_provider_get_status                (const dc_provider_t* prov
 void            dc_provider_unref                     (dc_provider_t* provider);
 
 
-/**
- * Return an HTTP(S) GET response.
- * This function can be used to download remote content for HTML emails.
- *
- * @memberof dc_context_t
- * @param context The context object to take proxy settings from.
- * @param url HTTP or HTTPS URL.
- * @return The response must be released using dc_http_response_unref() after usage.
- *     NULL is returned on errors.
- */
-dc_http_response_t*     dc_get_http_response      (const dc_context_t* context, const char* url);
-
-
-/**
- * @class dc_http_response_t
- *
- * An object containing an HTTP(S) GET response.
- * Created by dc_get_http_response().
- */
-
-
-/**
- * Returns HTTP response MIME type as a string, e.g. "text/plain" or "text/html".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_mimetype (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response encoding, e.g. "utf-8".
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The string which must be released using dc_str_unref() after usage. May be NULL.
- */
-char*                   dc_http_response_get_encoding (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response contents.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob which must be released using dc_str_unref() after usage. NULL is never returned.
- */
-uint8_t*                dc_http_response_get_blob     (const dc_http_response_t* response);
-
-/**
- * Returns HTTP response content size.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- * @return The blob size.
- */
-size_t                  dc_http_response_get_size     (const dc_http_response_t* response);
-
-/**
- * Free an HTTP response object.
- *
- * @memberof dc_http_response_t
- * @param response HTTP response as returned by dc_get_http_response().
- */
-void                    dc_http_response_unref        (const dc_http_response_t* response);
-
-
 /**
  * @class dc_lot_t
  *
@@ -5347,6 +5298,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 /**
  * @class dc_reactions_t
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  *
  * An object representing all reactions for a single message.
  */
@@ -5354,6 +5306,7 @@ int64_t         dc_lot_get_timestamp     (const dc_lot_t* lot);
 /**
  * Returns array of contacts which reacted to the given message.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @return array of contact IDs. Use dc_array_get_cnt() to get array length and
@@ -5365,6 +5318,7 @@ dc_array_t*     dc_reactions_get_contacts(dc_reactions_t* reactions);
 /**
  * Returns a string containing space-separated reactions of a single contact.
  *
+ * @deprecated 2023-11-27, use jsonrpc method `get_message_reactions` instead
  * @memberof dc_reactions_t
  * @param reactions The object containing message reactions.
  * @param contact_id ID of the contact.
@@ -5380,6 +5334,7 @@ char*           dc_reactions_get_by_contact_id(dc_reactions_t* reactions, uint32
  *
  * Reactions objects are created by dc_get_msg_reactions().
  *
+ * @deprecated 2023-11-27
  * @memberof dc_reactions_t
  * @param reactions The object to free.
  *     If NULL is given, nothing is done.
@@ -6264,6 +6219,18 @@ void dc_event_unref(dc_event_t* event);
 #define DC_EVENT_SELFAVATAR_CHANGED               2110
 
 
+/**
+ * A multi-device synced config value changed. Maybe the app needs to refresh smth. For uniformity
+ * this is emitted on the source device too. The value isn't reported, otherwise it would be logged
+ * which might not be good for privacy. You can get the new value with
+ * `dc_get_config(context, data2)`.
+ *
+ * @param data1 0
+ * @param data2 (char*) Configuration key.
+ */
+#define DC_EVENT_CONFIG_SYNCED                    2111
+
+
 /**
  * webxdc status update received.
  * To get the received status update, use dc_get_webxdc_status_updates() with
@@ -6433,22 +6400,27 @@ void dc_event_unref(dc_event_t* event);
 /**
  * Download not needed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_DONE          0
+#define DC_DOWNLOAD_DONE           0
 
 /**
  * Download available, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_AVAILABLE    10
+#define DC_DOWNLOAD_AVAILABLE      10
 
 /**
  * Download failed, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_FAILURE      20
+#define DC_DOWNLOAD_FAILURE        20
+
+/**
+ * Download not needed, see dc_msg_get_download_state() for details.
+ */
+#define DC_DOWNLOAD_UNDECIPHERABLE 30
 
 /**
  * Download in progress, see dc_msg_get_download_state() for details.
  */
-#define DC_DOWNLOAD_IN_PROGRESS  1000
+#define DC_DOWNLOAD_IN_PROGRESS    1000
 
 
 
@@ -6613,7 +6585,7 @@ void dc_event_unref(dc_event_t* event);
 /// - %1$s will be replaced by the name of the verified contact
 #define DC_STR_CONTACT_VERIFIED           35
 
-/// "Cannot verify %1$s."
+/// "Cannot establish guaranteed end-to-end encryption with %1$s."
 ///
 /// Used in status messages.
 /// - %1$s will be replaced by the name of the contact that cannot be verified
@@ -7051,6 +7023,8 @@ void dc_event_unref(dc_event_t* event);
 /// "You added member %1$s."
 ///
 /// Used in status messages.
+///
+/// `%1$s` will be replaced by the added member's name.
 #define DC_STR_ADD_MEMBER_BY_YOU 128
 
 /// "Member %1$s added by %2$s."
@@ -7272,6 +7246,21 @@ void dc_event_unref(dc_event_t* event);
 /// Used as the first info messages in newly created groups.
 #define DC_STR_NEW_GROUP_SEND_FIRST_MESSAGE 172
 
+/// "Member %1$s added."
+///
+/// Used as info messages.
+///
+/// `%1$s` will be replaced by the added member's name.
+#define DC_STR_MESSAGE_ADD_MEMBER 173
+
+/// "Your email provider %1$s requires end-to-end encryption which is not setup yet."
+///
+/// Used as info messages when a message cannot be sent because it cannot be encrypted.
+///
+/// `%1$s` will be replaced by the provider's domain.
+#define DC_STR_INVALID_UNENCRYPTED_MAIL 174
+
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -31,7 +31,6 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
 use deltachat::message::MsgId;
-use deltachat::net::read_url_blob;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
@@ -557,6 +556,7 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::SecurejoinJoinerProgress { .. } => 2061,
         EventType::ConnectivityChanged => 2100,
         EventType::SelfavatarChanged => 2110,
+        EventType::ConfigSynced { .. } => 2111,
         EventType::WebxdcStatusUpdate { .. } => 2120,
         EventType::WebxdcInstanceDeleted { .. } => 2121,
     }
@@ -584,6 +584,7 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
         | EventType::Error(_)
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. }
         | EventType::IncomingMsgBunch { .. }
         | EventType::ErrorSelfNotInGroup(_) => 0,
         EventType::MsgsChanged { chat_id, .. }
@@ -644,7 +645,8 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ConnectivityChanged
         | EventType::WebxdcInstanceDeleted { .. }
         | EventType::IncomingMsgBunch { .. }
-        | EventType::SelfavatarChanged => 0,
+        | EventType::SelfavatarChanged
+        | EventType::ConfigSynced { .. } => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
         | EventType::ReactionsChanged { msg_id, .. }
@@ -723,6 +725,10 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
             .to_c_string()
             .unwrap_or_default()
             .into_raw(),
+        EventType::ConfigSynced { key } => {
+            let data2 = key.to_string().to_c_string().unwrap_or_default();
+            data2.into_raw()
+        }
     }
 }
 
@@ -4113,10 +4119,26 @@ pub unsafe extern "C" fn dc_contact_is_verified(contact: *mut dc_contact_t) -> l
     let ffi_contact = &*contact;
     let ctx = &*ffi_contact.context;
 
-    block_on(ffi_contact.contact.is_verified(ctx))
+    if block_on(ffi_contact.contact.is_verified(ctx))
         .context("is_verified failed")
         .log_err(ctx)
-        .unwrap_or_default() as libc::c_int
+        .unwrap_or_default()
+    {
+        // Return value is essentially a boolean,
+        // but we return 2 for true for backwards compatibility.
+        2
+    } else {
+        0
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_is_bot(contact: *mut dc_contact_t) -> libc::c_int {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_is_bot()");
+        return 0;
+    }
+    (*contact).contact.is_bot() as libc::c_int
 }
 
 #[no_mangle]
@@ -4590,96 +4612,6 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
     // this may change once we start localizing string.
 }
 
-// dc_http_response_t
-
-pub type dc_http_response_t = net::HttpResponse;
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_get_http_response(
-    context: *const dc_context_t,
-    url: *const libc::c_char,
-) -> *mut dc_http_response_t {
-    if context.is_null() || url.is_null() {
-        eprintln!("ignoring careless call to dc_get_http_response()");
-        return ptr::null_mut();
-    }
-
-    let context = &*context;
-    let url = to_string_lossy(url);
-    if let Ok(response) = block_on(read_url_blob(context, &url))
-        .context("read_url_blob")
-        .log_err(context)
-    {
-        Box::into_raw(Box::new(response))
-    } else {
-        ptr::null_mut()
-    }
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_mimetype(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_mimetype()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.mimetype.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_encoding(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_encoding()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    response.encoding.strdup()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_blob(
-    response: *const dc_http_response_t,
-) -> *mut libc::c_char {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_blob()");
-        return ptr::null_mut();
-    }
-
-    let response = &*response;
-    let blob_len = response.blob.len();
-    let ptr = libc::malloc(blob_len);
-    libc::memcpy(ptr, response.blob.as_ptr() as *mut libc::c_void, blob_len);
-    ptr as *mut libc::c_char
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_get_size(
-    response: *const dc_http_response_t,
-) -> libc::size_t {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_get_size()");
-        return 0;
-    }
-
-    let response = &*response;
-    response.blob.len()
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_http_response_unref(response: *mut dc_http_response_t) {
-    if response.is_null() {
-        eprintln!("ignoring careless call to dc_http_response_unref()");
-        return;
-    }
-    drop(Box::from_raw(response));
-}
-
 // -- Accounts
 
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.131.2"
+version = "1.133.1"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"
@@ -17,11 +17,11 @@ deltachat = { path = ".." }
 num-traits = "0.2"
 schemars = "0.8.13"
 serde = { version = "1.0", features = ["derive"] }
-tempfile = "3.8.0"
+tempfile = "3.9.0"
 log = "0.4"
 async-channel = { version = "2.0.0" }
-futures = { version = "0.3.28" }
-serde_json = "1.0.105"
+futures = { version = "0.3.30" }
+serde_json = "1"
 yerpc = { version = "0.5.2", features = ["anyhow_expose", "openrpc"] }
 typescript-type-def = { version = "0.5.8", features = ["json_value"] }
 tokio = { version = "1.33.0" }
@@ -30,7 +30,7 @@ walkdir = "2.3.3"
 base64 = "0.21"
 
 # optional dependencies
-axum = { version = "0.6.20", optional = true, features = ["ws"] }
+axum = { version = "0.7", optional = true, features = ["ws"] }
 env_logger = { version = "0.10.0", optional = true }
 
 [dev-dependencies]

deltachat-jsonrpc/src/api.rs

@@ -15,7 +15,6 @@ use deltachat::constants::DC_MSG_ID_DAYMARKER;
 use deltachat::contact::{may_be_valid_addr, Contact, ContactId, Origin};
 use deltachat::context::get_info;
 use deltachat::ephemeral::Timer;
-use deltachat::imex;
 use deltachat::location;
 use deltachat::message::get_msg_read_receipts;
 use deltachat::message::{
@@ -28,6 +27,7 @@ use deltachat::reaction::{get_msg_reactions, send_reaction};
 use deltachat::securejoin;
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
+use deltachat::{imex, webxdc};
 use sanitize_filename::is_sanitized;
 use tokio::fs;
 use tokio::sync::{watch, Mutex, RwLock};
@@ -678,7 +678,7 @@ impl CommandApi {
     ///     the Verified-Group-Invite protocol is offered in the QR code;
     ///     works for protected groups as well as for normal groups.
     ///     If not set, the Setup-Contact protocol is offered in the QR code.
-    ///     See https://countermitm.readthedocs.io/en/latest/new.html
+    ///     See https://securejoin.readthedocs.io/en/latest/new.html
     ///     for details about both protocols.
     ///
     /// return format: `[code, svg]`
@@ -707,7 +707,7 @@ impl CommandApi {
     ///
     /// Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
     ///
-    /// See https://countermitm.readthedocs.io/en/latest/new.html
+    /// See https://securejoin.readthedocs.io/en/latest/new.html
     /// for details about both protocols.
     ///
     /// **qr**: The text of the scanned QR code. Typically, the same string as given
@@ -1675,6 +1675,16 @@ impl CommandApi {
             .await
     }
 
+    async fn join_gossip_topic(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+        topic: String,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        webxdc::join_gossip_topic(&ctx, MsgId::new(instance_msg_id), &topic).await
+    }
+
     async fn get_webxdc_status_updates(
         &self,
         account_id: u32,

deltachat-jsonrpc/src/api/types/chat.rs

@@ -85,7 +85,7 @@ impl FullChat {
         let can_send = chat.can_send(context).await?;
 
         let was_seen_recently = if chat.get_type() == Chattype::Single {
-            match contact_ids.get(0) {
+            match contact_ids.first() {
                 Some(contact) => Contact::get_by_id(context, *contact)
                     .await
                     .context("failed to load contact for was_seen_recently")?

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -102,7 +102,7 @@ pub(crate) async fn get_chat_list_item_by_id(
     let self_in_group = chat_contacts.contains(&ContactId::SELF);
 
     let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
-        let contact = chat_contacts.get(0);
+        let contact = chat_contacts.first();
         let was_seen_recently = match contact {
             Some(contact) => Contact::get_by_id(ctx, *contact)
                 .await

deltachat-jsonrpc/src/api/types/contact.rs

@@ -1,5 +1,4 @@
 use anyhow::Result;
-use deltachat::contact::VerifiedStatus;
 use deltachat::context::Context;
 use serde::Serialize;
 use typescript_type_def::TypeDef;
@@ -46,6 +45,9 @@ pub struct ContactObject {
     /// the contact's last seen timestamp
     last_seen: i64,
     was_seen_recently: bool,
+
+    /// If the contact is a bot.
+    is_bot: bool,
 }
 
 impl ContactObject {
@@ -57,7 +59,7 @@ impl ContactObject {
             Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
             None => None,
         };
-        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+        let is_verified = contact.is_verified(context).await?;
         let is_profile_verified = contact.is_profile_verified(context).await?;
 
         let verifier_id = contact
@@ -81,6 +83,7 @@ impl ContactObject {
             verifier_id,
             last_seen: contact.last_seen(),
             was_seen_recently: contact.was_seen_recently(),
+            is_bot: contact.is_bot(),
         })
     }
 }

deltachat-jsonrpc/src/api/types/events.rs

@@ -28,55 +28,37 @@ pub enum EventType {
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Info {
-        msg: String,
-    },
+    Info { msg: String },
 
     /// Emitted when SMTP connection is established and login was successful.
-    SmtpConnected {
-        msg: String,
-    },
+    SmtpConnected { msg: String },
 
     /// Emitted when IMAP connection is established and login was successful.
-    ImapConnected {
-        msg: String,
-    },
+    ImapConnected { msg: String },
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    SmtpMessageSent {
-        msg: String,
-    },
+    SmtpMessageSent { msg: String },
 
     /// Emitted when an IMAP message has been marked as deleted
-    ImapMessageDeleted {
-        msg: String,
-    },
+    ImapMessageDeleted { msg: String },
 
     /// Emitted when an IMAP message has been moved
-    ImapMessageMoved {
-        msg: String,
-    },
+    ImapMessageMoved { msg: String },
 
     /// Emitted before going into IDLE on the Inbox folder.
     ImapInboxIdle,
 
     /// Emitted when an new file in the $BLOBDIR was created
-    NewBlobFile {
-        file: String,
-    },
+    NewBlobFile { file: String },
 
     /// Emitted when an file in the $BLOBDIR was deleted
-    DeletedBlobFile {
-        file: String,
-    },
+    DeletedBlobFile { file: String },
 
     /// The library-user should write a warning string to the log.
     ///
     /// This event should *not* be reported to the end-user using a popup or something like
     /// that.
-    Warning {
-        msg: String,
-    },
+    Warning { msg: String },
 
     /// The library-user should report an error to the end-user.
     ///
@@ -88,18 +70,14 @@ pub enum EventType {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    Error {
-        msg: String,
-    },
+    Error { msg: String },
 
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// setChatName(), setChatProfileImage(),
     /// addContactToChat(), removeContactFromChat(),
     /// and messages sending functions.
-    ErrorSelfNotInGroup {
-        msg: String,
-    },
+    ErrorSelfNotInGroup { msg: String },
 
     /// Messages or chats changed.  One or more messages or chats changed for various
     /// reasons in the database:
@@ -110,10 +88,7 @@ pub enum EventType {
     /// `chatId` is set if only a single chat is affected by the changes, otherwise 0.
     /// `msgId` is set if only a single message is affected by the changes, otherwise 0.
     #[serde(rename_all = "camelCase")]
-    MsgsChanged {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgsChanged { chat_id: u32, msg_id: u32 },
 
     /// Reactions for the message changed.
     #[serde(rename_all = "camelCase")]
@@ -128,10 +103,7 @@ pub enum EventType {
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
     #[serde(rename_all = "camelCase")]
-    IncomingMsg {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    IncomingMsg { chat_id: u32, msg_id: u32 },
 
     /// Downloading a bunch of messages just finished. This is an experimental
     /// event to allow the UI to only show one notification per message bunch,
@@ -139,47 +111,31 @@ pub enum EventType {
     ///
     /// msg_ids contains the message ids.
     #[serde(rename_all = "camelCase")]
-    IncomingMsgBunch {
-        msg_ids: Vec<u32>,
-    },
+    IncomingMsgBunch { msg_ids: Vec<u32> },
 
     /// Messages were seen or noticed.
     /// chat id is always set.
     #[serde(rename_all = "camelCase")]
-    MsgsNoticed {
-        chat_id: u32,
-    },
+    MsgsNoticed { chat_id: u32 },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgDelivered {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDelivered { chat_id: u32, msg_id: u32 },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgFailed {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgFailed { chat_id: u32, msg_id: u32 },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see `Message.state`.
     #[serde(rename_all = "camelCase")]
-    MsgRead {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgRead { chat_id: u32, msg_id: u32 },
 
     /// A single message is deleted.
     #[serde(rename_all = "camelCase")]
-    MsgDeleted {
-        chat_id: u32,
-        msg_id: u32,
-    },
+    MsgDeleted { chat_id: u32, msg_id: u32 },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -189,24 +145,17 @@ pub enum EventType {
     /// This event does not include ephemeral timer modification, which
     /// is a separate event.
     #[serde(rename_all = "camelCase")]
-    ChatModified {
-        chat_id: u32,
-    },
+    ChatModified { chat_id: u32 },
 
     /// Chat ephemeral timer changed.
     #[serde(rename_all = "camelCase")]
-    ChatEphemeralTimerModified {
-        chat_id: u32,
-        timer: u32,
-    },
+    ChatEphemeralTimerModified { chat_id: u32, timer: u32 },
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
     #[serde(rename_all = "camelCase")]
-    ContactsChanged {
-        contact_id: Option<u32>,
-    },
+    ContactsChanged { contact_id: Option<u32> },
 
     /// Location of one or more contact has changed.
     ///
@@ -214,9 +163,7 @@ pub enum EventType {
     ///     If the locations of several contacts have been changed,
     ///     this parameter is set to `None`.
     #[serde(rename_all = "camelCase")]
-    LocationChanged {
-        contact_id: Option<u32>,
-    },
+    LocationChanged { contact_id: Option<u32> },
 
     /// Inform about the configuration progress started by configure().
     ConfigureProgress {
@@ -234,9 +181,7 @@ pub enum EventType {
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexProgress {
-        progress: usize,
-    },
+    ImexProgress { progress: usize },
 
     /// A file has been exported. A file has been written by imex().
     /// This event may be sent multiple times by a single call to imex().
@@ -246,9 +191,7 @@ pub enum EventType {
     ///
     /// @param data2 0
     #[serde(rename_all = "camelCase")]
-    ImexFileWritten {
-        path: String,
-    },
+    ImexFileWritten { path: String },
 
     /// Progress information of a secure-join handshake from the view of the inviter
     /// (Alice, the person who shows the QR code).
@@ -263,10 +206,7 @@ pub enum EventType {
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
     #[serde(rename_all = "camelCase")]
-    SecurejoinInviterProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
     /// Progress information of a secure-join handshake from the view of the joiner
     /// (Bob, the person who scans the QR code).
@@ -277,10 +217,7 @@ pub enum EventType {
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[serde(rename_all = "camelCase")]
-    SecurejoinJoinerProgress {
-        contact_id: u32,
-        progress: usize,
-    },
+    SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// The connectivity to the server changed.
     /// This means that you should refresh the connectivity view
@@ -288,8 +225,17 @@ pub enum EventType {
     /// getConnectivityHtml() for details.
     ConnectivityChanged,
 
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: String,
+    },
+
     #[serde(rename_all = "camelCase")]
     WebxdcStatusUpdate {
         msg_id: u32,
@@ -298,9 +244,7 @@ pub enum EventType {
 
     /// Inform that a message containing a webxdc instance has been deleted
     #[serde(rename_all = "camelCase")]
-    WebxdcInstanceDeleted {
-        msg_id: u32,
-    },
+    WebxdcInstanceDeleted { msg_id: u32 },
 }
 
 impl From<CoreEventType> for EventType {
@@ -396,6 +340,9 @@ impl From<CoreEventType> for EventType {
             },
             CoreEventType::ConnectivityChanged => ConnectivityChanged,
             CoreEventType::SelfavatarChanged => SelfavatarChanged,
+            CoreEventType::ConfigSynced { key } => ConfigSynced {
+                key: key.to_string(),
+            },
             CoreEventType::WebxdcStatusUpdate {
                 msg_id,
                 status_update_serial,

deltachat-jsonrpc/src/api/types/message.rs

@@ -345,6 +345,7 @@ pub enum SystemMessageType {
     SecurejoinMessage,
     LocationStreamingEnabled,
     LocationOnly,
+    InvalidUnencryptedMail,
 
     /// Chat ephemeral message timer is changed.
     EphemeralTimerChanged,
@@ -385,6 +386,7 @@ impl From<deltachat::mimeparser::SystemMessage> for SystemMessageType {
             SystemMessage::MultiDeviceSync => SystemMessageType::MultiDeviceSync,
             SystemMessage::WebxdcStatusUpdate => SystemMessageType::WebxdcStatusUpdate,
             SystemMessage::WebxdcInfoMessage => SystemMessageType::WebxdcInfoMessage,
+            SystemMessage::InvalidUnencryptedMail => SystemMessageType::InvalidUnencryptedMail,
         }
     }
 }

deltachat-jsonrpc/src/webserver.rs

@@ -33,10 +33,8 @@ async fn main() -> Result<(), std::io::Error> {
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));
     log::info!("JSON-RPC WebSocket server listening on {}", addr);
-    axum::Server::bind(&addr)
-        .serve(app.into_make_service())
-        .await
-        .unwrap();
+    let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+    axum::serve(listener, app).await.unwrap();
 
     Ok(())
 }

deltachat-jsonrpc/typescript/package.json

@@ -53,5 +53,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.131.2"
+  "version": "1.133.1"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.131.2"
+version = "1.133.1"
 license = "MPL-2.0"
 edition = "2021"
 
@@ -12,7 +12,7 @@ dirs = "5"
 log = "0.4.20"
 pretty_env_logger = "0.5"
 rusqlite = "0.30"
-rustyline = "12"
+rustyline = "13"
 tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
 
 [features]

deltachat-repl/src/cmdline.rs

@@ -284,13 +284,8 @@ async fn log_contactlist(context: &Context, contacts: &[ContactId]) -> Result<()
         let contact = Contact::get_by_id(context, *contact_id).await?;
         let name = contact.get_display_name();
         let addr = contact.get_addr();
-        let verified_state = contact.is_verified(context).await?;
-        let verified_str = if VerifiedStatus::Unverified != verified_state {
-            if verified_state == VerifiedStatus::BidirectVerified {
-                " √√"
-            } else {
-                " √"
-            }
+        let verified_str = if contact.is_verified(context).await? {
+            " √"
         } else {
             ""
         };

deltachat-repl/src/main.rs

@@ -299,8 +299,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize) -> bool {
-        self.highlighter.highlight_char(line, pos)
+    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
+        self.highlighter.highlight_char(line, pos, forced)
     }
 }
 
@@ -481,7 +481,10 @@ async fn handle_cmd(
 
 #[tokio::main]
 async fn main() -> Result<(), Error> {
-    let _ = pretty_env_logger::try_init();
+    pretty_env_logger::formatted_timed_builder()
+        .parse_default_env()
+        .filter_module("deltachat_repl", log::LevelFilter::Info)
+        .init();
 
     let args = std::env::args().collect();
     start(args).await?;

deltachat-rpc-client/examples/echobot_no_hooks.py

@@ -40,13 +40,13 @@ def main():
 
         while True:
             event = account.wait_for_event()
-            if event["type"] == EventType.INFO:
+            if event["kind"] == EventType.INFO:
                 logging.info("%s", event["msg"])
-            elif event["type"] == EventType.WARNING:
+            elif event["kind"] == EventType.WARNING:
                 logging.warning("%s", event["msg"])
-            elif event["type"] == EventType.ERROR:
+            elif event["kind"] == EventType.ERROR:
                 logging.error("%s", event["msg"])
-            elif event["type"] == EventType.INCOMING_MSG:
+            elif event["kind"] == EventType.INCOMING_MSG:
                 logging.info("Got an incoming message")
                 process_messages()
 

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -218,7 +218,7 @@ class Account:
         The function returns immediately and the handshake runs in background, sending
         and receiving several messages.
         Subsequent calls of `secure_join()` will abort previous, unfinished handshakes.
-        See https://countermitm.readthedocs.io/en/latest/new.html for protocol details.
+        See https://securejoin.readthedocs.io/en/latest/new.html for protocol details.
 
         :param qrdata: The text of the scanned QR code.
         """
@@ -271,6 +271,18 @@ class Account:
             if event.kind == EventType.INCOMING_MSG:
                 return event
 
+    def wait_for_securejoin_inviter_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
+                break
+
+    def wait_for_securejoin_joiner_success(self):
+        while True:
+            event = self.wait_for_event()
+            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
+                break
+
     def get_fresh_messages_in_arrival_order(self) -> List[Message]:
         """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
         warn(
@@ -288,3 +300,13 @@ class Account:
     def import_backup(self, path, passphrase: str = "") -> None:
         """Import backup."""
         self._rpc.import_backup(self.id, str(path), passphrase)
+
+    def export_self_keys(self, path) -> None:
+        """Export keys."""
+        passphrase = ""  # Setting passphrase is currently not supported.
+        self._rpc.export_self_keys(self.id, str(path), passphrase)
+
+    def import_self_keys(self, path) -> None:
+        """Import keys."""
+        passphrase = ""  # Importing passphrase-protected keys is currently not supported.
+        self._rpc.import_self_keys(self.id, str(path), passphrase)

deltachat-rpc-client/src/deltachat_rpc_client/client.py

@@ -195,7 +195,7 @@ class Client:
 
 
 class Bot(Client):
-    """Simple bot implementation that listent to events of a single account."""
+    """Simple bot implementation that listens to events of a single account."""
 
     def configure(self, email: str, password: str, **kwargs) -> None:
         kwargs.setdefault("bot", "1")

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -42,6 +42,10 @@ class Message:
             return AttrDict(reactions)
         return None
 
+    def get_sender_contact(self) -> Contact:
+        from_id = self.get_snapshot().from_id
+        return self.account.get_contact_by_id(from_id)
+
     def mark_seen(self) -> None:
         """Mark the message as seen."""
         self._rpc.markseen_msgs(self.account.id, [self.id])

deltachat-rpc-client/tests/test_securejoin.py

@@ -1,58 +1,73 @@
 import logging
 
+import pytest
 from deltachat_rpc_client import Chat, SpecialContactId
 
 
-def test_qr_setup_contact(acfactory) -> None:
+def test_qr_setup_contact(acfactory, tmp_path) -> None:
     alice, bob = acfactory.get_online_accounts(2)
 
     qr_code, _svg = alice.get_qr_code()
     bob.secure_join(qr_code)
 
-    while True:
-        event = alice.wait_for_event()
-        if event["kind"] == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    alice.wait_for_securejoin_inviter_success()
 
     # Test that Alice verified Bob's profile.
     alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
     alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
     assert alice_contact_bob_snapshot.is_verified
 
-    while True:
-        event = bob.wait_for_event()
-        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
-            break
+    bob.wait_for_securejoin_joiner_success()
 
     # Test that Bob verified Alice's profile.
     bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
     bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
     assert bob_contact_alice_snapshot.is_verified
 
+    # Test that if Bob changes the key, backwards verification is lost.
+    logging.info("Bob 2 is created")
+    bob2 = acfactory.new_configured_account()
+    bob2.export_self_keys(tmp_path)
 
-def test_qr_securejoin(acfactory):
+    logging.info("Bob imports a key")
+    bob.import_self_keys(tmp_path / "private-key-default.asc")
+
+    assert bob.get_config("key_id") == "2"
+    bob_contact_alice_snapshot = bob_contact_alice.get_snapshot()
+    assert not bob_contact_alice_snapshot.is_verified
+
+
+@pytest.mark.parametrize("protect", [True, False])
+def test_qr_securejoin(acfactory, protect):
     alice, bob = acfactory.get_online_accounts(2)
 
     logging.info("Alice creates a verified group")
-    alice_chat = alice.create_group("Verified group", protect=True)
+    alice_chat = alice.create_group("Verified group", protect=protect)
+    assert alice_chat.get_basic_snapshot().is_protected == protect
 
     logging.info("Bob joins verified group")
     qr_code, _svg = alice_chat.get_qr_code()
     bob.secure_join(qr_code)
-    while True:
-        event = alice.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+
+    # Check that at least some of the handshake messages are deleted.
+    for ac in [alice, bob]:
+        while True:
+            event = ac.wait_for_event()
+            if event["kind"] == "ImapMessageDeleted":
+                break
+
+    alice.wait_for_securejoin_inviter_success()
 
     # Test that Alice verified Bob's profile.
     alice_contact_bob = alice.get_contact_by_addr(bob.get_config("addr"))
     alice_contact_bob_snapshot = alice_contact_bob.get_snapshot()
     assert alice_contact_bob_snapshot.is_verified
 
-    while True:
-        event = bob.wait_for_event()
-        if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
-            break
+    bob.wait_for_securejoin_joiner_success()
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Member Me ({}) added by {}.".format(bob.get_config("addr"), alice.get_config("addr"))
+    assert snapshot.chat.get_basic_snapshot().is_protected == protect
 
     # Test that Bob verified Alice's profile.
     bob_contact_alice = bob.get_contact_by_addr(alice.get_config("addr"))
@@ -97,10 +112,7 @@ def test_qr_readreceipt(acfactory) -> None:
     charlie.secure_join(qr_code)
 
     for joiner in [bob, charlie]:
-        while True:
-            event = joiner.wait_for_event()
-            if event["kind"] == "SecurejoinJoinerProgress" and event["progress"] == 1000:
-                break
+        joiner.wait_for_securejoin_joiner_success()
 
     logging.info("Alice creates a verified group")
     group = alice.create_group("Group", protect=True)
@@ -152,7 +164,8 @@ def test_qr_readreceipt(acfactory) -> None:
     assert not bob.get_chat_by_contact(bob_contact_charlie)
 
 
-def test_verified_group_recovery(acfactory, rpc) -> None:
+def test_verified_group_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifying a member and sending a message in a group."""
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)
 
     logging.info("ac1 creates verified group")
@@ -162,10 +175,7 @@ def test_verified_group_recovery(acfactory, rpc) -> None:
     logging.info("ac2 joins verified group")
     qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -173,10 +183,8 @@ def test_verified_group_recovery(acfactory, rpc) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
@@ -184,70 +192,44 @@ def test_verified_group_recovery(acfactory, rpc) -> None:
     logging.info("ac2 reverifies with ac3")
     qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    while True:
-        event = ac3.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
     ac3_chat.send_text("Hi!")
 
-    msg_id = ac2.wait_for_incoming_msg_event().msg_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("Received message %s", snapshot.text)
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Hi!"
 
-    # ac1 contact cannot be verified by ac2 because ac3 did not gossip ac1 key in the "Hi!" message.
-    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
-    assert not ac1_contact.get_snapshot().is_verified
-
-    ac3_contact_id_ac1 = rpc.lookup_contact_id_by_addr(ac3.id, ac1.get_config("addr"))
-    ac3_chat.remove_contact(ac3_contact_id_ac1)
-    ac3_chat.add_contact(ac3_contact_id_ac1)
-
     msg_id = ac2.wait_for_incoming_msg_event().msg_id
     message = ac2.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "removed" in snapshot.text
-
-    event = ac2.wait_for_incoming_msg_event()
-    msg_id = event.msg_id
-    chat_id = event.chat_id
-    message = ac2.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
-    logging.info("ac2 got event message: %s", snapshot.text)
-    assert "added" in snapshot.text
+    assert snapshot.text == "Hi!"
 
+    # ac1 contact is verified for ac2 because ac3 gossiped ac1 key in the "Hi!" message.
+    ac1_contact = ac2.get_contact_by_addr(ac1.get_config("addr"))
     assert ac1_contact.get_snapshot().is_verified
 
-    chat = Chat(ac2, chat_id)
-    chat.send_text("Works again!")
+    # ac2 can write messages to the group.
+    snapshot.chat.send_text("Works again!")
 
-    msg_id = ac3.wait_for_incoming_msg_event().msg_id
-    message = ac3.get_message_by_id(msg_id)
-    snapshot = message.get_snapshot()
+    snapshot = ac3.get_message_by_id(ac3.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Works again!"
 
-    ac1.wait_for_incoming_msg_event()  # Hi!
-    ac1.wait_for_incoming_msg_event()  # Member removed
-    ac1.wait_for_incoming_msg_event()  # Member added
     snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
     assert snapshot.text == "Works again!"
 
-    # ac2 is now verified by ac3 for ac1
-    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
-    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
-
     ac1_chat_messages = snapshot.chat.get_messages()
     ac2_addr = ac2.get_config("addr")
     assert ac1_chat_messages[-2].get_snapshot().text == f"Changed setup for {ac2_addr}"
 
+    # ac2 is now verified by ac3 for ac1
+    ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
+    assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
 
 def test_verified_group_member_added_recovery(acfactory) -> None:
+    """Tests verified group recovery by reverifiying than removing and adding a member back."""
     ac1, ac2, ac3 = acfactory.get_online_accounts(3)
 
     logging.info("ac1 creates verified group")
@@ -257,10 +239,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     logging.info("ac2 joins verified group")
     qr_code, _svg = chat.get_qr_code()
     ac2.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac2.wait_for_securejoin_joiner_success()
 
     # ac1 has ac2 directly verified.
     ac1_contact_ac2 = ac1.get_contact_by_addr(ac2.get_config("addr"))
@@ -268,10 +247,8 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
 
     logging.info("ac3 joins verified group")
     ac3_chat = ac3.secure_join(qr_code)
-    while True:
-        event = ac1.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac3.wait_for_securejoin_joiner_success()
+    ac3.wait_for_incoming_msg_event()  # Member added
 
     logging.info("ac2 logs in on a new device")
     ac2 = acfactory.resetup_account(ac2)
@@ -279,11 +256,7 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     logging.info("ac2 reverifies with ac3")
     qr_code, _svg = ac3.get_qr_code()
     ac2.secure_join(qr_code)
-
-    while True:
-        event = ac3.wait_for_event()
-        if event.kind == "SecurejoinInviterProgress" and event["progress"] == 1000:
-            break
+    ac2.wait_for_securejoin_joiner_success()
 
     logging.info("ac3 sends a message to the group")
     assert len(ac3_chat.get_contacts()) == 3
@@ -339,3 +312,255 @@ def test_verified_group_member_added_recovery(acfactory) -> None:
     # ac2 is now verified by ac3 for ac1
     ac1_contact_ac3 = ac1.get_contact_by_addr(ac3.get_config("addr"))
     assert ac1_contact_ac2.get_snapshot().verifier_id == ac1_contact_ac3.id
+
+
+def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory):
+    """Regression test for
+    issue <https://github.com/deltachat/deltachat-core-rust/issues/4894>.
+    """
+    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
+
+    logging.info("ac3: verify with ac2")
+    qr_code, _svg = ac2.get_qr_code()
+    ac3.secure_join(qr_code)
+    ac2.wait_for_securejoin_inviter_success()
+
+    # in order for ac2 to have pending bobstate with a verified group
+    # we first create a fully joined verified group, and then start
+    # joining a second time but interrupt it, to create pending bob state
+
+    logging.info("ac1: create verified group that ac2 fully joins")
+    ch1 = ac1.create_group("Group", protect=True)
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    # ensure ac1 can write and ac2 receives messages in verified chat
+    ch1.send_text("ac1 says hello")
+    while 1:
+        snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if snapshot.text == "ac1 says hello":
+            assert snapshot.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
+    qr_code, _svg = ch1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac1.remove()
+    logging.info("ac2 now has pending bobstate but ac1 is shutoff")
+
+    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
+    assert ac3.get_contact_by_addr(ac2.get_config("addr")).get_snapshot().is_verified
+    assert ac2.get_contact_by_addr(ac3.get_config("addr")).get_snapshot().is_verified
+
+    logging.info("ac3: create a verified group VG with ac2")
+    vg = ac3.create_group("ac3-created", protect=True)
+    vg.add_contact(ac3.get_contact_by_addr(ac2.get_config("addr")))
+
+    # ensure ac2 receives message in VG
+    vg.send_text("hello")
+    while 1:
+        msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+        if msg.text == "hello":
+            assert msg.chat.get_basic_snapshot().is_protected
+            break
+
+    logging.info("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
+    qr_code, _svg = vg.get_qr_code()
+    ac4.secure_join(qr_code)
+    ac3.wait_for_securejoin_inviter_success()
+    while 1:
+        ev = ac2.wait_for_event()
+        if "added by unrelated SecureJoin" in str(ev):
+            return
+
+
+def test_qr_new_group_unblocked(acfactory):
+    """Regression test for a bug introduced in core v1.113.0.
+    ac2 scans a verified group QR code created by ac1.
+    This results in creation of a blocked 1:1 chat with ac1 on ac2,
+    but ac1 contact is not blocked on ac2.
+    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
+    ac2 should receive a message and create a contact request for the group.
+    Due to a bug previously ac2 created a blocked group.
+    """
+
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1_chat = ac1.create_group("Group for joining", protect=True)
+    qr_code, _svg = ac1_chat.get_qr_code()
+    ac2.secure_join(qr_code)
+
+    ac1.wait_for_securejoin_inviter_success()
+
+    ac1_new_chat = ac1.create_group("Another group")
+    ac1_new_chat.add_contact(ac1.get_contact_by_addr(ac2.get_config("addr")))
+    # Receive "Member added" message.
+    ac2.wait_for_incoming_msg_event()
+
+    ac1_new_chat.send_text("Hello!")
+    ac2_msg = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert ac2_msg.text == "Hello!"
+    assert ac2_msg.chat.get_basic_snapshot().is_contact_request
+
+
+def test_aeap_flow_verified(acfactory):
+    """Test that a new address is added to a contact when it changes its address."""
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+
+    logging.info("ac1: create verified-group QR, ac2 scans and joins")
+    chat = ac1.create_group("hello", protect=True)
+    assert chat.get_basic_snapshot().is_protected
+    qr_code, _svg = chat.get_qr_code()
+    logging.info("ac2: start QR-code based join-group protocol")
+    ac2.secure_join(qr_code)
+    ac1.wait_for_securejoin_inviter_success()
+
+    logging.info("sending first message")
+    msg_out = chat.send_text("old address").get_snapshot()
+
+    logging.info("receiving first message")
+    ac2.wait_for_incoming_msg_event()  # member added message
+    msg_in_1 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert msg_in_1.text == msg_out.text
+
+    logging.info("changing email account")
+    ac1.set_config("addr", ac1new.get_config("addr"))
+    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
+    ac1.stop_io()
+    ac1.configure()
+    ac1.start_io()
+
+    logging.info("sending second message")
+    msg_out = chat.send_text("changed address").get_snapshot()
+
+    logging.info("receiving second message")
+    msg_in_2 = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id)
+    msg_in_2_snapshot = msg_in_2.get_snapshot()
+    assert msg_in_2_snapshot.text == msg_out.text
+    assert msg_in_2_snapshot.chat.id == msg_in_1.chat.id
+    assert msg_in_2.get_sender_contact().get_snapshot().address == ac1new.get_config("addr")
+    assert len(msg_in_2_snapshot.chat.get_contacts()) == 2
+    assert ac1new.get_config("addr") in [
+        contact.get_snapshot().address for contact in msg_in_2_snapshot.chat.get_contacts()
+    ]
+
+
+def test_gossip_verification(acfactory) -> None:
+    alice, bob, carol = acfactory.get_online_accounts(3)
+
+    # Bob verifies Alice.
+    qr_code, _svg = alice.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    # Bob verifies Carol.
+    qr_code, _svg = carol.get_qr_code()
+    bob.secure_join(qr_code)
+    bob.wait_for_securejoin_joiner_success()
+
+    bob_contact_alice = bob.create_contact(alice.get_config("addr"), "Alice")
+    bob_contact_carol = bob.create_contact(carol.get_config("addr"), "Carol")
+    carol_contact_alice = carol.create_contact(alice.get_config("addr"), "Alice")
+
+    logging.info("Bob creates an Autocrypt group")
+    bob_group_chat = bob.create_group("Autocrypt Group")
+    assert not bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Autocrypt group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Autocrypt group"
+    assert snapshot.show_padlock
+
+    # Autocrypt group does not propagate verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert not carol_contact_alice_snapshot.is_verified
+
+    logging.info("Bob creates a Securejoin group")
+    bob_group_chat = bob.create_group("Securejoin Group", protect=True)
+    assert bob_group_chat.get_basic_snapshot().is_protected
+    bob_group_chat.add_contact(bob_contact_alice)
+    bob_group_chat.add_contact(bob_contact_carol)
+    bob_group_chat.send_message(text="Hello Securejoin group")
+
+    snapshot = carol.get_message_by_id(carol.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Securejoin group"
+    assert snapshot.show_padlock
+
+    # Securejoin propagates verification.
+    carol_contact_alice_snapshot = carol_contact_alice.get_snapshot()
+    assert carol_contact_alice_snapshot.is_verified
+
+
+def test_securejoin_after_contact_resetup(acfactory) -> None:
+    """
+    Regression test for a bug that prevented joining verified group with a QR code
+    if the group is already created and contains
+    a contact with inconsistent (Autocrypt and verified keys exist but don't match) key state.
+    """
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+
+    # ac3 creates protected group with ac1.
+    ac3_chat = ac3.create_group("Verified group", protect=True)
+
+    # ac1 joins ac3 group.
+    ac3_qr_code, _svg = ac3_chat.get_qr_code()
+    ac1.secure_join(ac3_qr_code)
+    ac1.wait_for_securejoin_joiner_success()
+
+    # ac1 waits for member added message and creates a QR code.
+    snapshot = ac1.get_message_by_id(ac1.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    ac1_qr_code, _svg = snapshot.chat.get_qr_code()
+
+    # ac2 verifies ac1
+    qr_code, _svg = ac1.get_qr_code()
+    ac2.secure_join(qr_code)
+    ac2.wait_for_securejoin_joiner_success()
+
+    # ac1 is verified for ac2.
+    ac2_contact_ac1 = ac2.create_contact(ac1.get_config("addr"), "")
+    assert ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 resetups the account.
+    ac1 = acfactory.resetup_account(ac1)
+
+    # ac1 sends a message to ac2.
+    ac1_contact_ac2 = ac1.create_contact(ac2.get_config("addr"), "")
+    ac1_chat_ac2 = ac1_contact_ac2.create_chat()
+    ac1_chat_ac2.send_text("Hello!")
+
+    # ac2 receives a message.
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello!"
+
+    # ac1 is no longer verified for ac2 as new Autocrypt key is not the same as old verified key.
+    assert not ac2_contact_ac1.get_snapshot().is_verified
+
+    # ac1 goes offline.
+    ac1.remove()
+
+    # Scanning a QR code results in creating an unprotected group with an inviter.
+    # In this case inviter is ac1 which has an inconsistent key state.
+    # Normally inviter becomes verified as a result of Securejoin protocol
+    # and then the group chat becomes verified when "Member added" is received,
+    # but in this case ac1 is offline and this Securejoin process will never finish.
+    logging.info("ac2 scans ac1 QR code, this is not expected to finish")
+    ac2.secure_join(ac1_qr_code)
+
+    logging.info("ac2 scans ac3 QR code")
+    ac2.secure_join(ac3_qr_code)
+
+    logging.info("ac2 waits for joiner success")
+    ac2.wait_for_securejoin_joiner_success()
+
+    # Wait for member added.
+    logging.info("ac2 waits for member added message")
+    snapshot = ac2.get_message_by_id(ac2.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.is_info
+    ac2_chat = snapshot.chat
+    assert ac2_chat.get_basic_snapshot().is_protected
+    assert len(ac2_chat.get_contacts()) == 3
+
+    # ac1 is still "not verified" for ac2 due to inconsistent state.
+    assert not ac2_contact_ac1.get_snapshot().is_verified

deltachat-rpc-client/tests/test_something.py

@@ -140,12 +140,9 @@ def test_chat(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
     assert snapshot.chat_id == chat_id
@@ -224,12 +221,9 @@ def test_message(acfactory) -> None:
     alice_chat_bob = alice_contact_bob.create_chat()
     alice_chat_bob.send_text("Hello!")
 
-    while True:
-        event = bob.wait_for_event()
-        if event.kind == EventType.INCOMING_MSG:
-            chat_id = event.chat_id
-            msg_id = event.msg_id
-            break
+    event = bob.wait_for_incoming_msg_event()
+    chat_id = event.chat_id
+    msg_id = event.msg_id
 
     message = bob.get_message_by_id(msg_id)
     snapshot = message.get_snapshot()
@@ -331,7 +325,7 @@ def test_wait_next_messages(acfactory) -> None:
         next_messages_task = executor.submit(bot.wait_next_messages)
 
         bot_addr = bot.get_config("addr")
-        alice_contact_bot = alice.create_contact(bot_addr, "Bob")
+        alice_contact_bot = alice.create_contact(bot_addr, "Bot")
         alice_chat_bot = alice_contact_bot.create_chat()
         alice_chat_bot.send_text("Hello!")
 
@@ -341,7 +335,7 @@ def test_wait_next_messages(acfactory) -> None:
         assert snapshot.text == "Hello!"
 
 
-def test_import_export(acfactory, tmp_path) -> None:
+def test_import_export_backup(acfactory, tmp_path) -> None:
     alice = acfactory.new_configured_account()
     alice.export_backup(tmp_path)
 
@@ -352,6 +346,31 @@ def test_import_export(acfactory, tmp_path) -> None:
     assert alice2.manager.get_system_info()
 
 
+def test_import_export_keys(acfactory, tmp_path) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    alice_chat_bob.send_text("Hello Bob!")
+
+    snapshot = bob.get_message_by_id(bob.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Bob!"
+
+    # Alice resetups account, but keeps the key.
+    alice_keys_path = tmp_path / "alice_keys"
+    alice_keys_path.mkdir()
+    alice.export_self_keys(alice_keys_path)
+    alice = acfactory.resetup_account(alice)
+    alice.import_self_keys(alice_keys_path)
+
+    snapshot.chat.accept()
+    snapshot.chat.send_text("Hello Alice!")
+    snapshot = alice.get_message_by_id(alice.wait_for_incoming_msg_event().msg_id).get_snapshot()
+    assert snapshot.text == "Hello Alice!"
+    assert snapshot.show_padlock
+
+
 def test_openrpc_command_line() -> None:
     """Test that "deltachat-rpc-server --openrpc" command returns an OpenRPC specification."""
     out = subprocess.run(["deltachat-rpc-server", "--openrpc"], capture_output=True, check=True).stdout
@@ -377,3 +396,46 @@ def test_provider_info(rpc) -> None:
     rpc.set_config(account_id, "socks5_enabled", "1")
     provider_info = rpc.get_provider_info(account_id, "github.com")
     assert provider_info is None
+
+
+def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+
+    # Bob creates chat manually so chat with Alice is accepted.
+    alice_chat_bob = alice_contact_bob.create_chat()
+
+    # Alice sends a message to Bob.
+    alice_chat_bob.send_text("Hello Bob!")
+    event = bob.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = bob.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+
+    # Bob sends a message to Alice.
+    bob_chat_alice = snapshot.chat
+    bob_chat_alice.accept()
+    bob_chat_alice.send_text("Hello Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock
+
+    # Alice reads Bob's message.
+    message.mark_seen()
+    while True:
+        event = bob.wait_for_event()
+        if event.kind == EventType.MSG_READ:
+            break
+
+    # Bob sends a message to Alice, it should also be encrypted.
+    bob_chat_alice.send_text("Hi Alice!")
+    event = alice.wait_for_incoming_msg_event()
+    msg_id = event.msg_id
+    message = alice.get_message_by_id(msg_id)
+    snapshot = message.get_snapshot()
+    assert snapshot.show_padlock

deltachat-rpc-client/tests/test_webxdc.py

@@ -43,3 +43,15 @@ def test_webxdc(acfactory) -> None:
     assert status_updates == [
         {"payload": "Second update", "serial": 2, "max_serial": 2},
     ]
+
+
+def test_webxdc_insert_lots_of_updates(acfactory) -> None:
+    alice, bob = acfactory.get_online_accounts(2)
+
+    bob_addr = bob.get_config("addr")
+    alice_contact_bob = alice.create_contact(bob_addr, "Bob")
+    alice_chat_bob = alice_contact_bob.create_chat()
+    message = alice_chat_bob.send_message(text="Let's play chess!", file="../test-data/webxdc/chess.xdc")
+
+    for i in range(2000):
+        message.send_webxdc_status_update({"payload": str(i)}, "description")

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.131.2"
+version = "1.133.1"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"
@@ -17,7 +17,7 @@ anyhow = "1"
 env_logger = { version = "0.10.0" }
 futures-lite = "2.0.0"
 log = "0.4"
-serde_json = "1.0.105"
+serde_json = "1"
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1.33.0", features = ["io-std"] }
 tokio-util = "0.7.9"

deny.toml

@@ -3,6 +3,13 @@ unmaintained = "allow"
 ignore = [
     "RUSTSEC-2020-0071",
     "RUSTSEC-2022-0093",
+
+    # Timing attack on RSA.
+    # Delta Chat does not use RSA for new keys
+    # and this requires precise measurement of the decryption time by the attacker.
+    # There is no fix at the time of writing this (2023-11-28).
+    # <https://rustsec.org/advisories/RUSTSEC-2023-0071>
+    "RUSTSEC-2023-0071",
 ]
 
 [bans]
@@ -11,50 +18,58 @@ ignore = [
 # when upgrading.
 # Please keep this list alphabetically sorted.
 skip = [
-     { name = "async-channel", version = "1.9.0" },
-     { name = "base16ct", version = "0.1.1" },
-     { name = "base64", version = "<0.21" },
-     { name = "bitflags", version = "1.3.2" },
-     { name = "block-buffer", version = "<0.10" },
-     { name = "convert_case", version = "0.4.0" },
-     { name = "curve25519-dalek", version = "3.2.0" },
-     { name = "darling_core", version = "<0.14" },
-     { name = "darling_macro", version = "<0.14" },
-     { name = "darling", version = "<0.14" },
-     { name = "der", version = "0.6.1" },
-     { name = "digest", version = "<0.10" },
-     { name = "ed25519-dalek", version = "1.0.1" },
-     { name = "ed25519", version = "1.5.3" },
-     { name = "event-listener", version = "2.5.3" },
-     { name = "getrandom", version = "<0.2" },
-     { name = "hashbrown", version = "<0.14.0" },
-     { name = "indexmap", version = "<2.0.0" },
-     { name = "pem-rfc7468", version = "0.6.0" },
-     { name = "pkcs8", version = "0.9.0" },
-     { name = "quick-error", version = "<2.0" },
-     { name = "rand_chacha", version = "<0.3" },
-     { name = "rand_core", version = "<0.6" },
-     { name = "rand", version = "<0.8" },
-     { name = "redox_syscall", version = "0.3.5" },
-     { name = "regex-automata", version = "0.1.10" },
-     { name = "regex-syntax", version = "0.6.29" },
-     { name = "ring", version = "0.16.20" },
-     { name = "sec1", version = "0.3.0" },
-     { name = "sha2", version = "<0.10" },
-     { name = "signature", version = "1.6.4" },
-     { name = "socket2", version = "0.4.9" },
-     { name = "spin", version = "<0.9.6" },
-     { name = "spki", version = "0.6.0" },
-     { name = "syn", version = "1.0.109" },
-     { name = "time", version = "<0.3" },
-     { name = "untrusted", version = "0.7.1" },
-     { name = "wasi", version = "<0.11" },
-     { name = "windows_aarch64_msvc", version = "<0.48" },
-     { name = "windows_i686_gnu", version = "<0.48" },
-     { name = "windows_i686_msvc", version = "<0.48" },
-     { name = "windows", version = "0.32.0" },
-     { name = "windows_x86_64_gnu", version = "<0.48" },
-     { name = "windows_x86_64_msvc", version = "<0.48" },
+    { name = "async-channel", version = "1.9.0" },
+    { name = "base16ct", version = "0.1.1" },
+    { name = "base64", version = "<0.21" },
+    { name = "bitflags", version = "1.3.2" },
+    { name = "block-buffer", version = "<0.10" },
+    { name = "convert_case", version = "0.4.0" },
+    { name = "curve25519-dalek", version = "3.2.0" },
+    { name = "darling_core", version = "<0.14" },
+    { name = "darling_macro", version = "<0.14" },
+    { name = "darling", version = "<0.14" },
+    { name = "der", version = "0.6.1" },
+    { name = "digest", version = "<0.10" },
+    { name = "ed25519-dalek", version = "1.0.1" },
+    { name = "ed25519", version = "1.5.3" },
+    { name = "event-listener", version = "2.5.3" },
+    { name = "fd-lock", version = "3.0.13" },
+    { name = "getrandom", version = "<0.2" },
+    { name = "h2", version = "0.3.22" },
+    { name = "http-body", version = "0.4.5" },
+    { name = "http", version = "0.2.11" },
+    { name = "hyper", version = "0.14.27" },
+    { name = "idna", version = "0.4.0" },
+    { name = "pem-rfc7468", version = "0.6.0" },
+    { name = "pkcs8", version = "0.9.0" },
+    { name = "quick-error", version = "<2.0" },
+    { name = "rand_chacha", version = "<0.3" },
+    { name = "rand_core", version = "<0.6" },
+    { name = "rand", version = "<0.8" },
+    { name = "redox_syscall", version = "0.3.5" },
+    { name = "regex-automata", version = "0.1.10" },
+    { name = "regex-syntax", version = "0.6.29" },
+    { name = "ring", version = "0.16.20" },
+    { name = "sec1", version = "0.3.0" },
+    { name = "sha2", version = "<0.10" },
+    { name = "signature", version = "1.6.4" },
+    { name = "socket2", version = "0.4.9" },
+    { name = "spin", version = "<0.9.6" },
+    { name = "spki", version = "0.6.0" },
+    { name = "syn", version = "1.0.109" },
+    { name = "time", version = "<0.3" },
+    { name = "untrusted", version = "0.7.1" },
+    { name = "wasi", version = "<0.11" },
+    { name = "windows_aarch64_gnullvm", version = "<0.52" },
+    { name = "windows_aarch64_msvc", version = "<0.52" },
+    { name = "windows_i686_gnu", version = "<0.52" },
+    { name = "windows_i686_msvc", version = "<0.52" },
+    { name = "windows-sys", version = "<0.52" },
+    { name = "windows-targets", version = "<0.52" },
+    { name = "windows", version = "0.32.0" },
+    { name = "windows_x86_64_gnullvm", version = "<0.52" },
+    { name = "windows_x86_64_gnu", version = "<0.52" },
+    { name = "windows_x86_64_msvc", version = "<0.52" },
 ]
 
 
@@ -64,7 +79,7 @@ allow = [
     "Apache-2.0",
     "BSD-2-Clause",
     "BSD-3-Clause",
-    "BSL-1.0", # Boost Software License 1.0
+    "BSL-1.0",          # Boost Software License 1.0
     "CC0-1.0",
     "ISC",
     "MIT",
@@ -77,13 +92,13 @@ allow = [
 [[licenses.clarify]]
 name = "ring"
 expression = "MIT AND ISC AND OpenSSL"
-license-files = [
-      { path = "LICENSE", hash = 0xbd0eed23 },
-]
+license-files = [{ path = "LICENSE", hash = 0xbd0eed23 }]
 
 [sources.allow-org]
 # Organisations which we allow git sources from.
 github = [
-       "async-email",
-       "deltachat",
+    "async-email",
+    "deltachat",
+    "djc",
+    "n0-computer", # iroh
 ]

node/constants.js

@@ -28,9 +28,11 @@ module.exports = {
   DC_DOWNLOAD_DONE: 0,
   DC_DOWNLOAD_FAILURE: 20,
   DC_DOWNLOAD_IN_PROGRESS: 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE: 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED: 2021,
   DC_EVENT_CHAT_MODIFIED: 2020,
   DC_EVENT_CONFIGURE_PROGRESS: 2041,
+  DC_EVENT_CONFIG_SYNCED: 2111,
   DC_EVENT_CONNECTIVITY_CHANGED: 2100,
   DC_EVENT_CONTACTS_CHANGED: 2030,
   DC_EVENT_DELETED_BLOB_FILE: 151,
@@ -78,6 +80,7 @@ module.exports = {
   DC_INFO_EPHEMERAL_TIMER_CHANGED: 10,
   DC_INFO_GROUP_IMAGE_CHANGED: 3,
   DC_INFO_GROUP_NAME_CHANGED: 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL: 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED: 8,
   DC_INFO_LOCATION_ONLY: 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP: 4,
@@ -224,11 +227,13 @@ module.exports = {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU: 124,
   DC_STR_IMAGE: 9,
   DC_STR_INCOMING_MESSAGES: 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL: 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY: 111,
   DC_STR_LOCATION: 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER: 137,
   DC_STR_LOCATION_ENABLED_BY_YOU: 136,
   DC_STR_MESSAGES: 114,
+  DC_STR_MESSAGE_ADD_MEMBER: 173,
   DC_STR_MSGACTIONBYME: 63,
   DC_STR_MSGACTIONBYUSER: 62,
   DC_STR_MSGADDMEMBER: 17,

node/events.js

@@ -34,6 +34,7 @@ module.exports = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED'
 }

node/lib/constants.ts

@@ -28,9 +28,11 @@ export enum C {
   DC_DOWNLOAD_DONE = 0,
   DC_DOWNLOAD_FAILURE = 20,
   DC_DOWNLOAD_IN_PROGRESS = 1000,
+  DC_DOWNLOAD_UNDECIPHERABLE = 30,
   DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED = 2021,
   DC_EVENT_CHAT_MODIFIED = 2020,
   DC_EVENT_CONFIGURE_PROGRESS = 2041,
+  DC_EVENT_CONFIG_SYNCED = 2111,
   DC_EVENT_CONNECTIVITY_CHANGED = 2100,
   DC_EVENT_CONTACTS_CHANGED = 2030,
   DC_EVENT_DELETED_BLOB_FILE = 151,
@@ -78,6 +80,7 @@ export enum C {
   DC_INFO_EPHEMERAL_TIMER_CHANGED = 10,
   DC_INFO_GROUP_IMAGE_CHANGED = 3,
   DC_INFO_GROUP_NAME_CHANGED = 2,
+  DC_INFO_INVALID_UNENCRYPTED_MAIL = 13,
   DC_INFO_LOCATIONSTREAMING_ENABLED = 8,
   DC_INFO_LOCATION_ONLY = 9,
   DC_INFO_MEMBER_ADDED_TO_GROUP = 4,
@@ -224,11 +227,13 @@ export enum C {
   DC_STR_GROUP_NAME_CHANGED_BY_YOU = 124,
   DC_STR_IMAGE = 9,
   DC_STR_INCOMING_MESSAGES = 103,
+  DC_STR_INVALID_UNENCRYPTED_MAIL = 174,
   DC_STR_LAST_MSG_SENT_SUCCESSFULLY = 111,
   DC_STR_LOCATION = 66,
   DC_STR_LOCATION_ENABLED_BY_OTHER = 137,
   DC_STR_LOCATION_ENABLED_BY_YOU = 136,
   DC_STR_MESSAGES = 114,
+  DC_STR_MESSAGE_ADD_MEMBER = 173,
   DC_STR_MSGACTIONBYME = 63,
   DC_STR_MSGACTIONBYUSER = 62,
   DC_STR_MSGADDMEMBER = 17,
@@ -318,6 +323,7 @@ export const EventId2EventName: { [key: number]: string } = {
   2061: 'DC_EVENT_SECUREJOIN_JOINER_PROGRESS',
   2100: 'DC_EVENT_CONNECTIVITY_CHANGED',
   2110: 'DC_EVENT_SELFAVATAR_CHANGED',
+  2111: 'DC_EVENT_CONFIG_SYNCED',
   2120: 'DC_EVENT_WEBXDC_STATUS_UPDATE',
   2121: 'DC_EVENT_WEBXDC_INSTANCE_DELETED',
 }

node/lib/deltachat.ts

@@ -178,7 +178,7 @@ export class AccountManager extends EventEmitter {
   static newTemporary() {
     let directory = null
     while (true) {
-      const randomString = Math.random().toString(36).substr(2, 5)
+      const randomString = Math.random().toString(36).substring(2, 5)
       directory = join(tmpdir(), 'deltachat-' + randomString)
       if (!existsSync(directory)) break
     }

node/test/test.mjs

@@ -1,13 +1,17 @@
 // @ts-check
-import DeltaChat from '../dist'
+import { DeltaChat } from '../dist/index.js'
 
 import { deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
-import { EventId2EventName, C } from '../dist/constants'
+import { EventId2EventName, C } from '../dist/constants.js'
 import { join } from 'path'
 import { statSync } from 'fs'
-import { Context } from '../dist/context'
+import { Context } from '../dist/context.js'
+import {fileURLToPath} from 'url';
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
 chai.use(chaiAsPromised)
 chai.config.truncateThreshold = 0 // Do not truncate assertion errors.
 

package.json

@@ -8,9 +8,8 @@
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "@types/node": "^20.8.10",
-    "chai": "^4.2.0",
+    "chai": "~4.3.10",
     "chai-as-promised": "^7.1.1",
-    "esm": "^3.2.25",
     "mocha": "^8.2.1",
     "node-gyp": "^10.0.0",
     "prebuildify": "^5.0.1",
@@ -53,8 +52,8 @@
     "prebuildify": "cd node && prebuildify -t 18.0.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
+    "test:mocha": "mocha node/test/test.mjs --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.131.2"
+  "version": "1.133.1"
 }

python/README.rst

@@ -1,6 +1,6 @@
-=========================
-DeltaChat Python bindings
-=========================
+============================
+CFFI Python Bindings
+============================
 
 This package provides `Python bindings`_ to the `deltachat-core library`_
 which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
@@ -8,157 +8,3 @@ a low-level Chat/Contact/Message API to user interfaces and bots.
 
 .. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
 .. _`Python bindings`: https://py.delta.chat/
-
-Installing pre-built packages (Linux-only)
-==========================================
-
-If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself`__.
-
-__ sourceinstall_
-
-We recommend to first create a fresh Python virtual environment
-and activate it in your shell::
-
-    python -m venv env
-    source env/bin/activate
-
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``env`` directory and leaves
-your system installation alone.
-
-For Linux we build wheels for all releases and push them to a python package
-index. To install the latest release::
-
-    pip install deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
-
-Running tests
-=============
-
-Recommended way to run tests is using `scripts/run-python-test.sh`
-script provided in the core repository.
-
-This script compiles the library in debug mode and runs the tests using `tox`_.
-By default it will run all "offline" tests and skip all functional
-end-to-end tests that require accounts on real e-mail servers.
-
-.. _`tox`: https://tox.wiki
-.. _livetests:
-
-Running "live" tests with temporary accounts
---------------------------------------------
-
-If you want to run live functional tests
-you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
-that creates e-mail accounts like this::
-
-    export CHATMAIL_DOMAIN=nine.testrun.org
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the server.
-These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
-After setting the variable, either rerun `scripts/run-python-test.sh`
-or run offline and online tests with `tox` directly::
-
-    tox -e py
-
-Each test run creates new accounts.
-
-Developing the bindings
------------------------
-
-If you want to develop or debug the bindings,
-you can create a testing development environment using `tox`::
-
-    export DCC_RS_DEV="$PWD"
-    export DCC_RS_TARGET=debug
-    tox -c python --devenv env -e py
-    . env/bin/activate
-
-Inside this environment the bindings are installed
-in editable mode (as if installed with `python -m pip install -e`)
-together with the testing dependencies like `pytest` and its plugins.
-
-You can then edit the source code in the development tree
-and quickly run `pytest` manually without waiting  for `tox`
-to recreating the virtual environment each time.
-
-.. _sourceinstall:
-
-Installing bindings from source
-===============================
-
-Install Rust and Cargo first.
-The easiest is probably to use `rustup <https://rustup.rs/>`_.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then clone the deltachat-core-rust repo::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-First, build the core library::
-
-   cargo build --release -p deltachat_ffi --features jsonrpc
-
-`jsonrpc` feature is required even if not used by the bindings
-because `deltachat.h` includes JSON-RPC functions unconditionally.
-
-Create the virtual environment and activate it:
-
-   python -m venv env
-   source env/bin/activate
-
-Build and install the bindings:
-
-   export DCC_RS_DEV="$PWD"
-   export DCC_RS_TARGET=release
-   python -m pip install ./python
-
-`DCC_RS_DEV` environment variable specifies the location of
-the core development tree. If this variable is not set,
-`libdeltachat` library and `deltachat.h` header are expected
-to be installed system-wide.
-
-When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
-the build profile name to look up the artifacts
-in the target directory.
-In this case setting it can be skipped because
-`DCC_RS_TARGET=release` is the default.
-
-Building manylinux based wheels
-===============================
-
-Building portable manylinux wheels which come with libdeltachat.so
-can be done with Docker_ or Podman_.
-
-.. _Docker: https://www.docker.com/
-.. _Podman: https://podman.io/
-
-If you want to build your own wheels, build container image first::
-
-   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
-   $ docker build -t deltachat/coredeps scripts/coredeps
-
-This will use the ``scripts/coredeps/Dockerfile`` to build
-container image called ``deltachat/coredeps``.  You can afterwards
-find it with::
-
-   $ docker images
-
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e CHATMAIL_DOMAIN \
-       --rm -it -v $(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh

python/doc/Makefile

@@ -1,197 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-VERSION       = $(shell python -c "import conf ; print(conf.version)")
-DOCZIP        = devpi-$(VERSION).doc.zip
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-PAPER         =
-BUILDDIR      = _build
-RSYNCOPTS     = -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
-
-export HOME=/tmp/home
-export TESTHOME=$(HOME)
-
-# Internal variables.
-PAPEROPT_a4     = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-# This variable is not auto generated as the order is important.
-USER_MAN_CHAPTERS = commands\
-                    user\
-                    indices\
-                    packages\
-#                 userman/index.rst\
-#                 userman/devpi_misc.rst\
-#                 userman/devpi_concepts.rst\
-
-
-#export DEVPI_CLIENTDIR=$(CURDIR)/.tmp_devpi_user_man/client
-#export DEVPI_SERVERDIR=$(CURDIR)/.tmp_devpi_user_man/server
-
-chapter = commands
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp \
-        epub latex latexpdf text man changes linkcheck doctest gettext install \
-        quickstart-releaseprocess quickstart-pypimirror quickstart-server regen \
-        prepare-quickstart\
-        regen.server-fresh regen.server-restart regen.server-clean\
-        regen.uman-all regen.uman
-
-help:
-	@echo "Please use \`make <target>' where <target> is one of"
-	@echo "  html       to make standalone HTML files"
-	@echo "  dirhtml    to make HTML files named index.html in directories"
-	@echo "  singlehtml to make a single large HTML file"
-	@echo "  pickle     to make pickle files"
-	@echo "  json       to make JSON files"
-	@echo "  htmlhelp   to make HTML files and a HTML help project"
-	@echo "  qthelp     to make HTML files and a qthelp project"
-	@echo "  devhelp    to make HTML files and a Devhelp project"
-	@echo "  epub       to make an epub"
-	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
-	@echo "  text       to make text files"
-	@echo "  man        to make manual pages"
-	@echo "  texinfo    to make Texinfo files"
-	@echo "  info       to make Texinfo files and run them through makeinfo"
-	@echo "  gettext    to make PO message catalogs"
-	@echo "  changes    to make an overview of all changed/added/deprecated items"
-	@echo "  linkcheck  to check all external links for integrity"
-	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
-	@echo
-	@echo "User Manual Regen Targets"
-	@echo "  regen.uman        regenerates page. of the user manual chapeter e.g. regen.uman chapter=..."
-	@echo "  regen.uman-all    regenerates the user manual"
-	@echo "  regen.uman-clean  stop temp server and clean up directory"
-	@echo "  Chapter List: $(USER_MAN_CHAPTERS)"
-
-clean:
-	-rm -rf $(BUILDDIR)/*
-
-version:
-	@echo "version $(VERSION)"
-
-doczip: html
-	python doczip.py $(DOCZIP) _build/html
-
-install: html
-	rsync -avz $(RSYNCOPTS) _build/html/ delta@py.delta.chat:build/master
-
-
-html:
-	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
-	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
-	@echo
-	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json:
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/devpi.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/devpi.qhc"
-
-devhelp:
-	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
-	@echo
-	@echo "Build finished."
-	@echo "To view the help file:"
-	@echo "# mkdir -p $$HOME/.local/share/devhelp/devpi"
-	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/devpi"
-	@echo "# devhelp"
-
-epub:
-	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
-	@echo
-	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make' in that directory to run these through (pdf)latex" \
-	      "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo "Running LaTeX files through pdflatex..."
-	$(MAKE) -C $(BUILDDIR)/latex all-pdf
-	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
-	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
-	@echo
-	@echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
-	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
-	@echo
-	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo
-	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
-	@echo "Run \`make' in that directory to run these through makeinfo" \
-	      "(use \`make info' here to do that automatically)."
-
-info:
-	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
-	@echo "Running Texinfo files through makeinfo..."
-	make -C $(BUILDDIR)/texinfo info
-	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
-	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
-	@echo
-	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
-	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
-	@echo
-	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
-

python/doc/_templates/globaltoc.html

@@ -1,17 +0,0 @@
-
-<div class="globaltoc">
-
-<ul>
-  <li><a href="{{ pathto('index') }}">index</a></li>
-  <li><a href="{{ pathto('install') }}">install</a></li>
-  <li><a href="{{ pathto('api') }}">high level API</a></li>
-  <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-</ul>
-<b>external links:</b>
-<ul>
-  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
-  <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
-  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
-</ul>
-
-</div>

python/doc/_templates/sidebarintro.html

@@ -1 +0,0 @@
-<h3>deltachat {{release}}</h3>

python/doc/cffi/api.rst

@@ -1,5 +1,4 @@
-
-high level API reference
+High Level API Reference
 ========================
 
 - :class:`deltachat.Account` (your main entry point, creates the
@@ -8,28 +7,14 @@ high level API reference
 - :class:`deltachat.Chat`
 - :class:`deltachat.Message`
 
-Account
--------
-
 .. autoclass:: deltachat.Account
-    :members:
-
-
-Contact
--------
+   :members:
 
 .. autoclass:: deltachat.Contact
-    :members:
-
-Chat
-----
+   :members:
 
 .. autoclass:: deltachat.Chat
-    :members:
-
-Message
--------
+   :members:
 
 .. autoclass:: deltachat.Message
-    :members:
-
+   :members:

python/doc/cffi/examples.rst

@@ -1,11 +1,10 @@
-
-examples
+Examples
 ========
 
 Once you have :doc:`installed deltachat bindings <install>`
 you need email/password credentials for an IMAP/SMTP account.
 Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+temporary email accounts on `testrun.org <https://testrun.org/>`_ for testing.
 
 Receiving a Chat message from the command line
 ----------------------------------------------
@@ -16,11 +15,11 @@ Here is a simple bot that:
 
 - terminates the bot if the message `/quit` is sent
 
-.. include:: ../examples/echo_and_quit.py
+.. include:: ../../examples/echo_and_quit.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     $ cd examples
@@ -40,11 +39,11 @@ Here is a simple bot that:
 
 - tracks member additions and removals for all chat groups
 
-.. include:: ../examples/group_tracking.py
+.. include:: ../../examples/group_tracking.py
     :literal:
 
 With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
+by specifying a database path, an email address and password of
 a SMTP-IMAP account::
 
     python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db

python/doc/cffi/install.rst

@@ -0,0 +1,80 @@
+Install
+=======
+
+Installing pre-built packages (Linux-only)
+------------------------------------------
+
+If you have a Linux system you may install the ``deltachat`` binary "wheel" packages
+without any "build-from-source" steps.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
+
+We recommend to first create a fresh Python virtual environment
+and activate it in your shell::
+
+    python -m venv env
+    source env/bin/activate
+
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``env`` directory and leaves
+your system installation alone.
+
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
+
+    pip install deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
+
+.. _sourceinstall:
+
+Installing bindings from source
+-------------------------------
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+First, build the core library::
+
+   cargo build --release -p deltachat_ffi --features jsonrpc
+
+`jsonrpc` feature is required even if not used by the bindings
+because `deltachat.h` includes JSON-RPC functions unconditionally.
+
+Create the virtual environment and activate it::
+
+   python -m venv env
+   source env/bin/activate
+
+Build and install the bindings::
+
+   export DCC_RS_DEV="$PWD"
+   export DCC_RS_TARGET=release
+   python -m pip install ./python
+
+`DCC_RS_DEV` environment variable specifies the location of
+the core development tree. If this variable is not set,
+`libdeltachat` library and `deltachat.h` header are expected
+to be installed system-wide.
+
+When `DCC_RS_DEV` is set, `DCC_RS_TARGET` specifies
+the build profile name to look up the artifacts
+in the target directory.
+In this case setting it can be skipped because
+`DCC_RS_TARGET=release` is the default.

python/doc/cffi/intro.rst

@@ -0,0 +1,11 @@
+Introduction
+============
+
+CFFI bindings are available via the `deltachat <https://pypi.org/project/deltachat/>`_ Python package.
+The package contains both the Python bindings and the Delta Chat core.
+It is provided only for Linux.
+
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
+low-level CFFI bindings to the C interface of the Delta Chat core
+and high-level Python bindings built on top of CFFI bindings.

python/doc/cffi/lapi.rst

@@ -1,8 +1,7 @@
+Low Level API Reference
+=======================
 
-low level API reference
-===================================
-
-for full doxygen-generated C-docs, defines and functions please checkout
+For full doxygen-generated C-docs, defines and functions please checkout
 
 https://c.delta.chat
 

python/doc/cffi/manylinux.rst

@@ -0,0 +1,25 @@
+Building Manylinux-Based Wheels
+===============================
+
+Building portable manylinux wheels which come with libdeltachat.so
+can be done with Docker_ or Podman_.
+
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
+
+If you want to build your own wheels, build container image first::
+
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
+
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
+find it with::
+
+   $ docker images
+
+This docker image can be used to run tests and build Python wheels for all interpreters::
+
+    $ docker run -e CHATMAIL_DOMAIN \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/doc/cffi/tests.rst

@@ -0,0 +1,49 @@
+Running Tests
+=============
+
+Recommended way to run tests is using `scripts/run-python-test.sh`
+script provided in the core repository.
+
+This script compiles the library in debug mode and runs the tests using `tox`_.
+By default it will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real email servers.
+
+.. _`tox`: https://tox.wiki
+.. _livetests:
+
+Running "Live" Tests With Temporary Accounts
+--------------------------------------------
+
+If you want to run live functional tests
+you can set ``CHATMAIL_DOMAIN`` to a domain of the email server
+that creates email accounts like this::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+With this account-creation setting, pytest runs create ephemeral email accounts on the server.
+These accounts have the pattern `ci-{6 characters}@{CHATMAIL_DOMAIN}`.
+After setting the variable, either rerun `scripts/run-python-test.sh`
+or run offline and online tests with `tox` directly::
+
+    tox -e py
+
+Each test run creates new accounts.
+
+Developing the Bindings
+-----------------------
+
+If you want to develop or debug the bindings,
+you can create a testing development environment using `tox`::
+
+    export DCC_RS_DEV="$PWD"
+    export DCC_RS_TARGET=debug
+    tox -c python --devenv env -e py
+    . env/bin/activate
+
+Inside this environment the bindings are installed
+in editable mode (as if installed with `python -m pip install -e`)
+together with the testing dependencies like `pytest` and its plugins.
+
+You can then edit the source code in the development tree
+and quickly run `pytest` manually without waiting  for `tox`
+to recreating the virtual environment each time.

python/doc/changelog.rst

@@ -1,4 +0,0 @@
-Changelog for deltachat-core's Python bindings
-==============================================
-
-.. include:: ../CHANGELOG

python/doc/conf.py

@@ -1,138 +1,94 @@
-# -*- coding: utf-8 -*-
-#
-# devpi documentation build configuration file, created by
-# sphinx-quickstart on Mon Jun  3 16:11:22 2013.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import sys
-import os
+from pathlib import Path
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 from deltachat import __version__ as release
+
 version = ".".join(release.split(".")[:2])
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration -----------------------------------------------------
 
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.autosummary',
-    #'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx.ext.viewcode',
-    'breathe',
-    #'sphinx.ext.githubpages',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.viewcode",
+    "breathe",
+    "sphinx_rtd_theme",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix of source filenames.
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+project = "Delta Chat"
+copyright = "2023, Delta Chat contributors"
+author = "Delta Chat contributors"
 
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['sketch', '_build', "attic"]
+exclude_patterns = ["sketch", "_build", "attic", "Thumbs.db", ".DS_Store"]
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 
 # -- breathe options ------
 
-breathe_projects = {
-    "deltachat": "../../docs/xml/"
-}
+breathe_projects = {"deltachat": Path("../../docs/xml/")}
 
 breathe_default_project = "deltachat"
 
 # -- Options for HTML output ---------------------------------------------------
 
-sys.path.append(os.path.abspath('_themes'))
-html_theme_path = ['_themes']
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-# html_theme = 'flask'
-html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-html_theme_options = {
-    'logo': '_static/delta-chat.svg',
-    'font_size': "1.1em",
-    'caption_font_size': "0.9em",
-    'code_font_size': "1.1em",
-
-
-}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = ["_themes"]
+html_theme = "sphinx_rtd_theme"
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
@@ -141,51 +97,34 @@ html_logo = "_static/delta-chat.svg"
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = '_static/favicon.ico'
+html_favicon = "_static/favicon.ico"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#
-html_sidebars = {
-    'index': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'searchbox.html'
-    ],
-    '**': [
-        'sidebarintro.html',
-        'globaltoc.html',
-        'relations.html',
-        'searchbox.html'
-    ]
-}
-
+# html_use_smartypants = True
 
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
 html_show_sourcelink = False
@@ -194,71 +133,65 @@ html_show_sourcelink = False
 html_show_sphinx = False
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-html_use_opensearch = 'https://doc.devpi.net'
+html_use_opensearch = "https://doc.devpi.net"
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'deltachat-python'
+htmlhelp_basename = "deltachat-python"
 
 # -- Options for LaTeX output --------------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-'pointsize': '12pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '12pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'devpi.tex', u'deltachat documentation',
-   u'holger krekel', 'manual'),
+    ("index", "devpi.tex", "deltachat documentation", "holger krekel", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'deltachat', u'deltachat documentation',
-     [u'holger krekel'], 1)
-]
+man_pages = [("index", "deltachat", "deltachat documentation", ["holger krekel"], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output ------------------------------------------------
@@ -267,30 +200,38 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'devpi', u'devpi Documentation',
-   u'holger krekel', 'devpi', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        "index",
+        "devpi",
+        "devpi Documentation",
+        "holger krekel",
+        "devpi",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {"http://docs.python.org/": None}
 
 # autodoc options
 autodoc_member_order = "bysource"
+
+
 # always document __init__ functions
 def skip(app, what, name, obj, skip, options):
     return skip
 
+
 def setup(app):
     app.connect("autodoc-skip-member", skip)
-

python/doc/index.rst

@@ -1,41 +1,44 @@
-deltachat python bindings
-=========================
+Delta Chat Python bindings, new and old
+=======
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
-
-- :doc:`api` is a high level interface to deltachat-core.
-
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
-
-
-
-getting started
----------------
+`Delta Chat <https://delta.chat/>`_ provides two kinds of Python bindings
+to the `Rust Core <https://github.com/deltachat/deltachat-core-rust>`_:
+JSON-RPC bindings and CFFI bindings.
+When starting a new project it is recommended to use JSON-RPC bindings,
+which are used in the Delta Chat Desktop app through generated Typescript-bindings. 
+The Python JSON-RPC bindings are maintained by Delta Chat core developers. 
+Most existing bot projects and many tests in Delta Chat's own core library
+still use the CFFI-bindings, and it is going to be maintained certainly also in 2024. 
+New APIs might however only appear in the JSON-RPC bindings, 
+as the CFFI bindings are increasingly in maintenance-only mode. 
 
 .. toctree::
    :maxdepth: 2
+   :caption: JSON-RPC Bindings
 
-   install
-   examples
+   jsonrpc/intro
+   jsonrpc/install
+   jsonrpc/examples
+   jsonrpc/reference
+   jsonrpc/develop
 
 .. toctree::
-   :hidden:
+   :maxdepth: 2
+   :caption: CFFI Bindings
 
-   links
-   changelog
-   api
-   lapi
-   plugins
-
-..
-    Indices and tables
-    ==================
-
-    * :ref:`genindex`
-    * :ref:`modindex`
-    * :ref:`search`
+   cffi/intro
+   cffi/install
+   cffi/examples
+   cffi/manylinux
+   cffi/tests
+   cffi/api
+   cffi/lapi
+   cffi/plugins
 
+.. _`deltachat`: https://delta.chat
+.. _`deltachat-core repo`: https://github.com/deltachat
+.. _pip: http://pypi.org/project/pip/
+.. _virtualenv: http://pypi.org/project/virtualenv/
+.. _merlinux: http://merlinux.eu
+.. _pypi: http://pypi.org/
+.. _`issue-tracker`: https://github.com/deltachat/deltachat-core-rust

python/doc/install.rst

@@ -1,2 +0,0 @@
-
-.. include:: ../README.rst

python/doc/jsonrpc/develop.rst

@@ -0,0 +1,68 @@
+===========
+Development
+===========
+
+To develop JSON-RPC bindings,
+clone the `deltachat-core-rust <https://github.com/deltachat/deltachat-core-rust/>`_ repository::
+
+   git clone https://github.com/deltachat/deltachat-core-rust.git
+
+Testing
+=======
+
+To run online tests, set ``CHATMAIL_DOMAIN``
+to a domain of the email server
+that can be used to create testing accounts::
+
+    export CHATMAIL_DOMAIN=nine.testrun.org
+
+Then run ``scripts/run-rpc-test.sh``
+to build debug version of ``deltachat-rpc-server``
+and run ``deltachat-rpc-client`` tests
+in a separate virtual environment managed by `tox <https://tox.wiki/>`_.
+
+Development Environment
+=======================
+
+Creating a new virtual environment
+to run the tests each time
+as ``scripts/run-rpc-test.sh`` does is slow
+if you are changing the tests or the code
+and want to rerun the tests each time.
+
+If you are developing the tests,
+it is better to create a persistent virtual environment.
+You can do this by running ``scripts/make-rpc-testenv.sh``.
+This creates a virtual environment ``venv`` which you can then enter with::
+
+   . venv/bin/activate
+
+Then you can run the tests with
+
+::
+
+    pytest deltachat-rpc-client/tests/
+
+Refer to `pytest documentation <https://docs.pytest.org/>` for details.
+
+If make the changes to Delta Chat core
+or Python bindings, you can rebuild the environment by rerunning
+``scripts/make-rpc-testenv.sh``.
+It is ok to rebuild the activated environment this way,
+you do not need to deactivate or reactivate the environment each time.
+
+Using REPL
+==========
+
+Once you have a development environment,
+you can quickly test things in REPL::
+
+   $ python
+   >>> from deltachat_rpc_client import *
+   >>> rpc = Rpc()
+   >>> rpc.start()
+   >>> dc = DeltaChat(rpc)
+   >>> system_info = dc.get_system_info()
+   >>> system_info["level"]
+   'awesome'
+   >>> rpc.close()

python/doc/jsonrpc/examples.rst

@@ -0,0 +1,19 @@
+Examples
+========
+
+Echo bot
+--------
+.. include:: ../../../deltachat-rpc-client/examples/echobot_no_hooks.py
+    :literal:
+
+Echo bot with hooks
+-------------------
+.. include:: ../../../deltachat-rpc-client/examples/echobot.py
+    :literal:
+
+Advanced echo bot
+-----------------
+
+.. include:: ../../../deltachat-rpc-client/examples/echobot_advanced.py
+    :literal:
+

python/doc/jsonrpc/install.rst

@@ -0,0 +1,36 @@
+Install
+=======
+
+To use JSON-RPC bindings for Delta Chat core you will need
+a ``deltachat-rpc-server`` binary which provides Delta Chat core API over JSON-RPC
+and a ``deltachat-rpc-client`` Python package which is a JSON-RPC client that starts ``deltachat-rpc-server`` process and uses JSON-RPC API.
+
+`Create a virtual environment <https://docs.python.org/3/library/venv.html>`__ if you
+don’t have one already and activate it::
+
+   $ python -m venv venv
+   $ . venv/bin/activate
+
+Install ``deltachat-rpc-server``
+--------------------------------
+
+To get ``deltachat-rpc-server`` binary you have three options:
+
+1. Install ``deltachat-rpc-server`` from PyPI using ``pip install deltachat-rpc-server``.
+2. Build and install ``deltachat-rpc-server`` from source with ``cargo install --git https://github.com/deltachat/deltachat-core-rust/ deltachat-rpc-server``.
+3. Download prebuilt release from https://github.com/deltachat/deltachat-core-rust/releases and install it into ``PATH``.
+
+Check that ``deltachat-rpc-server`` is installed and can run::
+
+   $ deltachat-rpc-server --version
+   1.131.4
+
+Then install ``deltachat-rpc-client`` with ``pip install deltachat-rpc-client``.
+
+Install ``deltachat-rpc-client``
+--------------------------------
+
+To get ``deltachat-rpc-client`` Python library you can:
+
+1. Install ``deltachat-rpc-client`` from PyPI using ``pip install deltachat-rpc-client``.
+2. Install ``deltachat-rpc-client`` from source with ``pip install git+https://github.com/deltachat/deltachat-core-rust.git@main#subdirectory=deltachat-rpc-client``.

python/doc/jsonrpc/intro.rst

@@ -0,0 +1,8 @@
+Introduction
+============
+
+JSON-RPC bindings are available via the `deltachat-rpc-client <https://pypi.org/project/deltachat-rpc-client/>`_ Python package.
+This package provides only the Python bindings and requires ``deltachat-rpc-server`` binary to be installed.
+`deltachat-rpc-server <https://pypi.org/project/deltachat-rpc-server/>`_ package provides ``deltachat-rpc-server`` binary for Linux, Windows, macOS and Android.
+
+RPC client connects to standalone Delta Chat RPC server ``deltachat-rpc-server`` and provides Python interface to it.

python/doc/jsonrpc/reference.rst

@@ -0,0 +1,5 @@
+API Reference
+=============
+
+.. automodule:: deltachat_rpc_client
+   :members:

python/doc/links.rst

@@ -1,11 +0,0 @@
-
-links
-================================
-
-.. _`deltachat`: https://delta.chat
-.. _`deltachat-core repo`: https://github.com/deltachat
-.. _pip: http://pypi.org/project/pip/
-.. _virtualenv: http://pypi.org/project/virtualenv/
-.. _merlinux: http://merlinux.eu
-.. _pypi: http://pypi.org/
-.. _`issue-tracker`: https://github.com/deltachat/deltachat-core

python/doc/make.bat

@@ -1,190 +0,0 @@
-@ECHO OFF
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set BUILDDIR=_build
-set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
-set I18NSPHINXOPTS=%SPHINXOPTS% .
-if NOT "%PAPER%" == "" (
-	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
-	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
-)
-
-if "%1" == "" goto help
-
-if "%1" == "help" (
-	:help
-	echo.Please use `make ^<target^>` where ^<target^> is one of
-	echo.  html       to make standalone HTML files
-	echo.  dirhtml    to make HTML files named index.html in directories
-	echo.  singlehtml to make a single large HTML file
-	echo.  pickle     to make pickle files
-	echo.  json       to make JSON files
-	echo.  htmlhelp   to make HTML files and a HTML help project
-	echo.  qthelp     to make HTML files and a qthelp project
-	echo.  devhelp    to make HTML files and a Devhelp project
-	echo.  epub       to make an epub
-	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
-	echo.  text       to make text files
-	echo.  man        to make manual pages
-	echo.  texinfo    to make Texinfo files
-	echo.  gettext    to make PO message catalogs
-	echo.  changes    to make an overview over all changed/added/deprecated items
-	echo.  linkcheck  to check all external links for integrity
-	echo.  doctest    to run all doctests embedded in the documentation if enabled
-	goto end
-)
-
-if "%1" == "clean" (
-	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
-	del /q /s %BUILDDIR%\*
-	goto end
-)
-
-if "%1" == "html" (
-	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
-	goto end
-)
-
-if "%1" == "dirhtml" (
-	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
-	goto end
-)
-
-if "%1" == "singlehtml" (
-	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
-	goto end
-)
-
-if "%1" == "pickle" (
-	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the pickle files.
-	goto end
-)
-
-if "%1" == "json" (
-	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can process the JSON files.
-	goto end
-)
-
-if "%1" == "htmlhelp" (
-	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run HTML Help Workshop with the ^
-.hhp project file in %BUILDDIR%/htmlhelp.
-	goto end
-)
-
-if "%1" == "qthelp" (
-	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; now you can run "qcollectiongenerator" with the ^
-.qhcp project file in %BUILDDIR%/qthelp, like this:
-	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\devpi.qhcp
-	echo.To view the help file:
-	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\devpi.ghc
-	goto end
-)
-
-if "%1" == "devhelp" (
-	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished.
-	goto end
-)
-
-if "%1" == "epub" (
-	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The epub file is in %BUILDDIR%/epub.
-	goto end
-)
-
-if "%1" == "latex" (
-	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
-	goto end
-)
-
-if "%1" == "text" (
-	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The text files are in %BUILDDIR%/text.
-	goto end
-)
-
-if "%1" == "man" (
-	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The manual pages are in %BUILDDIR%/man.
-	goto end
-)
-
-if "%1" == "texinfo" (
-	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
-	goto end
-)
-
-if "%1" == "gettext" (
-	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
-	goto end
-)
-
-if "%1" == "changes" (
-	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.The overview file is in %BUILDDIR%/changes.
-	goto end
-)
-
-if "%1" == "linkcheck" (
-	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Link check complete; look for any errors in the above output ^
-or in %BUILDDIR%/linkcheck/output.txt.
-	goto end
-)
-
-if "%1" == "doctest" (
-	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
-	if errorlevel 1 exit /b 1
-	echo.
-	echo.Testing of doctests in the sources finished, look at the ^
-results in %BUILDDIR%/doctest/output.txt.
-	goto end
-)
-
-:end

python/src/deltachat/testplugin.py

@@ -10,6 +10,7 @@ import time
 import weakref
 import random
 from queue import Queue
+from threading import Event
 from typing import Callable, Dict, List, Optional, Set
 
 import pytest
@@ -590,6 +591,27 @@ class ACFactory:
         ac2.create_chat(ac1)
         return ac1.create_chat(ac2)
 
+    def get_protected_chat(self, ac1: Account, ac2: Account):
+        class SetupPlugin:
+            def __init__(self) -> None:
+                self.member_added = Event()
+
+            @account_hookimpl
+            def ac_member_added(self, chat: deltachat.Chat, contact, actor, message):
+                self.member_added.set()
+
+        setupplugin = SetupPlugin()
+        ac1.add_account_plugin(setupplugin)
+        chat = ac1.create_group_chat("Protected Group", verified=True)
+        qr = chat.get_join_qr()
+        ac2.qr_join_chat(qr)
+        setupplugin.member_added.wait()
+        msg = ac2.wait_next_incoming_message()
+        assert msg.text == "Messages are guaranteed to be end-to-end encrypted from now on."
+        msg = ac2.wait_next_incoming_message()
+        assert "Member Me " in msg.text and " added by " in msg.text
+        return chat
+
     def introduce_each_other(self, accounts, sending=True):
         to_wait = []
         for i, acc in enumerate(accounts):

python/tests/test_1_online.py

@@ -9,7 +9,7 @@ import pytest
 from imap_tools import AND, U
 
 import deltachat as dc
-from deltachat import account_hookimpl, Message, Chat
+from deltachat import account_hookimpl, Message
 from deltachat.tracker import ImexTracker
 
 
@@ -498,6 +498,26 @@ def test_forward_messages(acfactory, lp):
     assert not chat3.get_messages()
 
 
+def test_forward_encrypted_to_unencrypted(acfactory, lp):
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
+    chat = acfactory.get_protected_chat(ac1, ac2)
+
+    lp.sec("ac1: send encrypted message to ac2")
+    txt = "This should be encrypted"
+    chat.send_text(txt)
+    msg = ac2.wait_next_incoming_message()
+    assert msg.text == txt
+    assert msg.is_encrypted()
+
+    lp.sec("ac2: forward message to ac3 unencrypted")
+    unencrypted_chat = ac2.create_chat(ac3)
+    msg_id = msg.id
+    msg2 = unencrypted_chat.send_msg(msg)
+    assert msg2 == msg
+    assert msg.id != msg_id
+    assert not msg.is_encrypted()
+
+
 def test_forward_own_message(acfactory, lp):
     ac1, ac2 = acfactory.get_online_accounts(2)
     chat = acfactory.get_accepted_chat(ac1, ac2)
@@ -1658,128 +1678,6 @@ def test_ac_setup_message_twice(acfactory, lp):
     assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
 
-def test_qr_setup_contact(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-    qr = ac1.get_setup_contact_qr()
-
-    lp.sec("ac2: start QR-code based setup contact protocol")
-    ch = ac2.qr_setup_contact(qr)
-    assert ch.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-
-@pytest.mark.parametrize("verified_one_on_one_chats", [0, 1])
-def test_qr_join_chat(acfactory, lp, verified_one_on_one_chats):
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
-    ac2.set_config("verified_one_on_one_chats", verified_one_on_one_chats)
-
-    lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
-    chat = ac1.create_group_chat("hello")
-    qr = chat.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    ch = ac2.qr_join_chat(qr)
-    lp.sec("ac2: qr_join_chat() returned")
-    assert ch.id >= 10
-    # check that at least some of the handshake messages are deleted
-    ac1._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    msg = ac2._evtracker.wait_next_incoming_message()
-    assert msg.text == "Member Me ({}) added by {}.".format(ac2.get_config("addr"), ac1.get_config("addr"))
-
-    # ac1 reloads the chat.
-    chat = Chat(chat.account, chat.id)
-    assert not chat.is_protected()
-
-    # ac2 reloads the chat.
-    ch = Chat(ch.account, ch.id)
-    assert not ch.is_protected()
-
-
-def test_qr_join_chat_with_pending_bobstate_issue4894(acfactory, lp):
-    ac1, ac2, ac3, ac4 = acfactory.get_online_accounts(4)
-
-    lp.sec("ac3: verify with ac2")
-    ac3.qr_setup_contact(ac2.get_setup_contact_qr())
-    ac2._evtracker.wait_securejoin_inviter_progress(1000)
-
-    # in order for ac2 to have pending bobstate with a verified group
-    # we first create a fully joined verified group, and then start
-    # joining a second time but interrupt it, to create pending bob state
-
-    lp.sec("ac1: create verified group that ac2 fully joins")
-    ch1 = ac1.create_group_chat("ac1-shutoff group", verified=True)
-    ac2.qr_join_chat(ch1.get_join_qr())
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    # ensure ac1 can write and ac2 receives messages in verified chat
-    ch1.send_text("ac1 says hello")
-    while 1:
-        msg = ac2.wait_next_incoming_message()
-        if msg.text == "ac1 says hello":
-            assert msg.chat.is_protected()
-            break
-
-    lp.sec("ac1: let ac2 join again but shutoff ac1 in the middle of securejoin")
-    ac2.qr_join_chat(ch1.get_join_qr())
-    ac1.shutdown()
-    lp.sec("ac2 now has pending bobstate but ac1 is shutoff")
-
-    # we meanwhile expect ac3/ac2 verification started in the beginning to have completed
-    assert ac3.get_contact(ac2).is_verified()
-    assert ac2.get_contact(ac3).is_verified()
-
-    lp.sec("ac3: create a verified group VG with ac2")
-    vg = ac3.create_group_chat("ac3-created", [ac2], verified=True)
-
-    # ensure ac2 receives message in VG
-    vg.send_text("hello")
-    while 1:
-        msg = ac2.wait_next_incoming_message()
-        if msg.text == "hello":
-            assert msg.chat.is_protected()
-            break
-
-    lp.sec("ac3: create a join-code for group VG and let ac4 join, check that ac2 got it")
-    ac4.qr_join_chat(vg.get_join_qr())
-    ac3._evtracker.wait_securejoin_inviter_progress(1000)
-    while 1:
-        ev = ac2._evtracker.get()
-        if "added by unrelated SecureJoin" in str(ev):
-            return
-
-
-def test_qr_new_group_unblocked(acfactory, lp):
-    """Regression test for a bug intoduced in core v1.113.0.
-    ac2 scans a verified group QR code created by ac1.
-    This results in creation of a blocked 1:1 chat with ac1 on ac2,
-    but ac1 contact is not blocked on ac2.
-    Then ac1 creates a group, adds ac2 there and promotes it by sending a message.
-    ac2 should receive a message and create a contact request for the group.
-    Due to a bug previously ac2 created a blocked group.
-    """
-
-    ac1, ac2 = acfactory.get_online_accounts(2)
-    ac1_chat = ac1.create_group_chat("Group for joining", verified=True)
-    qr = ac1_chat.get_join_qr()
-    ac2.qr_join_chat(qr)
-
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    ac1_new_chat = ac1.create_group_chat("Another group")
-    ac1_new_chat.add_contact(ac2)
-    # Receive "Member added" message.
-    ac2._evtracker.wait_next_incoming_message()
-
-    ac1_new_chat.send_text("Hello!")
-    ac2_msg = ac2._evtracker.wait_next_incoming_message()
-    assert ac2_msg.text == "Hello!"
-    assert ac2_msg.chat.is_contact_request()
-
-
 def test_qr_email_capitalization(acfactory, lp):
     """Regression test for a bug
     that resulted in failure to propagate verification via gossip in a verified group
@@ -2101,6 +1999,32 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
 
 
+def test_all_work_done(acfactory, lp):
+    """
+    Tests that calling start_io() immediately followed by maybe_network()
+    and then waiting for all_work_done() reliably fetches the messages
+    delivered while account was offline.
+    In other words, connectivity should not change to a state
+    where all_work_done() returns true until IMAP connection goes idle.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+
+    ac1.stop_io()
+    ac1._evtracker.wait_for_connectivity(dc.const.DC_CONNECTIVITY_NOT_CONNECTED)
+
+    ac1.direct_imap.select_config_folder("inbox")
+    with ac1.direct_imap.idle() as idle1:
+        ac2.create_chat(ac1).send_text("Hi")
+        idle1.wait_for_new_message()
+
+    ac1.start_io()
+    ac1.maybe_network()
+    ac1._evtracker.wait_for_all_work_done()
+    msgs = ac1.create_chat(ac2).get_messages()
+    assert len(msgs) == 1
+    assert msgs[0].text == "Hi"
+
+
 def test_fetch_deleted_msg(acfactory, lp):
     """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
     hundreds of times, because uid_next was not updated.
@@ -2527,47 +2451,6 @@ def test_delete_deltachat_folder(acfactory):
     assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
-def test_aeap_flow_verified(acfactory, lp):
-    """Test that a new address is added to a contact when it changes its address."""
-    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
-
-    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
-    chat = ac1.create_group_chat("hello", verified=True)
-    assert chat.is_protected()
-    qr = chat.get_join_qr()
-    lp.sec("ac2: start QR-code based join-group protocol")
-    chat2 = ac2.qr_join_chat(qr)
-    assert chat2.id >= 10
-    ac1._evtracker.wait_securejoin_inviter_progress(1000)
-
-    lp.sec("sending first message")
-    msg_out = chat.send_text("old address")
-
-    lp.sec("receiving first message")
-    ac2._evtracker.wait_next_incoming_message()  # member added message
-    msg_in_1 = ac2._evtracker.wait_next_incoming_message()
-    assert msg_in_1.text == msg_out.text
-
-    lp.sec("changing email account")
-    ac1.set_config("addr", ac1new.get_config("addr"))
-    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
-    ac1.stop_io()
-    configtracker = ac1.configure()
-    configtracker.wait_finish()
-    ac1.start_io()
-
-    lp.sec("sending second message")
-    msg_out = chat.send_text("changed address")
-
-    lp.sec("receiving second message")
-    msg_in_2 = ac2._evtracker.wait_next_incoming_message()
-    assert msg_in_2.text == msg_out.text
-    assert msg_in_2.chat.id == msg_in_1.chat.id
-    assert msg_in_2.get_sender_contact().addr == ac1new.get_config("addr")
-    assert len(msg_in_2.chat.get_contacts()) == 2
-    assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()]
-
-
 def test_archived_muted_chat(acfactory, lp):
     """If an archived and muted chat receives a new message, DC_EVENT_MSGS_CHANGED for
     DC_CHAT_ID_ARCHIVED_LINK must be generated if the chat had only seen messages previously.

python/tests/test_4_lowlevel.py

@@ -156,6 +156,8 @@ def test_markseen_invalid_message_ids(acfactory):
     chat = contact1.create_chat()
     chat.send_text("one message")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    # Skip configuration-related warnings, but not errors.
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
     ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")

python/tox.ini

@@ -62,9 +62,9 @@ commands =
 [testenv:doc]
 changedir=doc
 deps =
-# Pinned due to incompatibility of breathe with sphinx 7.2: <https://github.com/breathe-doc/breathe/issues/943>
-    sphinx<=7.1.2
+    sphinx
     breathe
+    sphinx_rtd_theme
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
 

release-date.in

@@ -1 +1 @@
-2023-11-14
+2024-01-21

scripts/README.md

@@ -39,6 +39,8 @@ and an own build machine.
 
 - `android-rpc-server.sh` compiles binaries of `deltachat-rpc-server` using Android NDK.
 
+- `build-python-docs.sh` builds Python documentation into `dist/html/`.
+
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 

scripts/build-python-docs.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export DCC_RS_TARGET=debug
+export DCC_RS_DEV="$PWD"
+cargo build -p deltachat_ffi --features jsonrpc
+
+python3 -m venv venv
+venv/bin/pip install ./python
+venv/bin/pip install ./deltachat-rpc-client
+venv/bin/pip install sphinx breathe sphinx_rtd_theme
+venv/bin/pip install ./deltachat-rpc-client
+venv/bin/sphinx-build -b html -a python/doc/ dist/html

scripts/concourse/docs_wheels.yml

@@ -102,8 +102,6 @@ jobs:
             - name: deltachat-core-rust-release
               path: .
           outputs:
-            - name: py-docs
-              path: ./python/doc/_build/
             # Binary wheels
             - name: py-wheels
               path: ./python/.docker-tox/wheelhouse/
@@ -115,28 +113,6 @@ jobs:
               - |
                 scripts/run_all.sh
 
-      # Upload python docs to py.delta.chat
-      - task: upload-py-docs
-        config:
-          inputs:
-            - name: py-docs
-          image_resource:
-            type: registry-image
-            source:
-              repository: alpine
-          platform: linux
-          run:
-            path: sh
-            args:
-              - -ec
-              - |
-                apk add --no-cache rsync openssh-client
-                mkdir -p ~/.ssh
-                chmod 700 ~/.ssh
-                echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-                chmod 600 ~/.ssh/id_ed25519
-                rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
       # Upload x86_64 wheels and source packages
       - task: upload-wheels
         config:

scripts/run_all.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Build the Delta Chat Core Rust library, Python wheels and docs 
+# Build the Delta Chat Core Rust library and Python wheels
 
 set -e -x
 
@@ -34,9 +34,3 @@ unset CHATMAIL_DOMAIN
 tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"
-
-
-echo -----------------------
-echo generating python docs
-echo -----------------------
-tox --workdir "$TOXWORKDIR" -e doc

scripts/set_core_version.py

@@ -102,7 +102,7 @@ def main():
                 found = True
         if not found:
             raise SystemExit(
-                f"{changelog_name} contains no entry for version: {newversion}"
+                f"CHANGELOG.md contains no entry for version: {newversion}"
             )
 
     for toml_filename in toml_list:

src/accounts.rs

@@ -8,11 +8,14 @@ use anyhow::{ensure, Context as _, Result};
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
-use tokio::sync::oneshot;
 use tokio::task::JoinHandle;
-use tokio::time::{sleep, Duration};
 use uuid::Uuid;
 
+#[cfg(not(target_os = "ios"))]
+use tokio::sync::oneshot;
+#[cfg(not(target_os = "ios"))]
+use tokio::time::{sleep, Duration};
+
 use crate::context::Context;
 use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::stock_str::StockStrings;
@@ -303,6 +306,7 @@ impl Accounts {
 const CONFIG_NAME: &str = "accounts.toml";
 
 /// Lockfile name.
+#[cfg(not(target_os = "ios"))]
 const LOCKFILE_NAME: &str = "accounts.lock";
 
 /// Database file name.
@@ -338,22 +342,16 @@ impl Drop for Config {
 }
 
 impl Config {
-    /// Creates a new Config for `file`, but doesn't open/sync it.
-    async fn new_nosync(file: PathBuf, lock: bool) -> Result<Self> {
-        let dir = file.parent().context("Cannot get config file directory")?;
-        let inner = InnerConfig {
-            accounts: Vec::new(),
-            selected_account: 0,
-            next_id: 1,
-        };
-        if !lock {
-            let cfg = Self {
-                file,
-                inner,
-                lock_task: None,
-            };
-            return Ok(cfg);
-        }
+    #[cfg(target_os = "ios")]
+    async fn create_lock_task(_dir: PathBuf) -> Result<Option<JoinHandle<anyhow::Result<()>>>> {
+        // Do not lock accounts.toml on iOS.
+        // This results in 0xdead10cc crashes on suspend.
+        // iOS itself ensures that multiple instances of Delta Chat are not running.
+        Ok(None)
+    }
+
+    #[cfg(not(target_os = "ios"))]
+    async fn create_lock_task(dir: PathBuf) -> Result<Option<JoinHandle<anyhow::Result<()>>>> {
         let lockfile = dir.join(LOCKFILE_NAME);
         let mut lock = fd_lock::RwLock::new(fs::File::create(lockfile).await?);
         let (locked_tx, locked_rx) = oneshot::channel();
@@ -384,12 +382,32 @@ impl Config {
             rx.await?;
             Ok(())
         });
+        locked_rx.await?;
+        Ok(Some(lock_task))
+    }
+
+    /// Creates a new Config for `file`, but doesn't open/sync it.
+    async fn new_nosync(file: PathBuf, lock: bool) -> Result<Self> {
+        let dir = file.parent().context("Cannot get config file directory")?;
+        let inner = InnerConfig {
+            accounts: Vec::new(),
+            selected_account: 0,
+            next_id: 1,
+        };
+        if !lock {
+            let cfg = Self {
+                file,
+                inner,
+                lock_task: None,
+            };
+            return Ok(cfg);
+        }
+        let lock_task = Self::create_lock_task(dir.to_path_buf()).await?;
         let cfg = Self {
             file,
             inner,
-            lock_task: Some(lock_task),
+            lock_task,
         };
-        locked_rx.await?;
         Ok(cfg)
     }
 
@@ -406,11 +424,13 @@ impl Config {
     /// Takes a mutable reference because the saved file is a part of the `Config` state. This
     /// protects from parallel calls resulting to a wrong file contents.
     async fn sync(&mut self) -> Result<()> {
+        #[cfg(not(target_os = "ios"))]
         ensure!(!self
             .lock_task
             .as_ref()
             .context("Config is read-only")?
             .is_finished());
+
         let tmp_path = self.file.with_extension("toml.tmp");
         let mut file = fs::File::create(&tmp_path)
             .await
@@ -526,8 +546,12 @@ impl Config {
             }
             if self.inner.selected_account == id {
                 // reset selected account
-                self.inner.selected_account =
-                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                self.inner.selected_account = self
+                    .inner
+                    .accounts
+                    .first()
+                    .map(|e| e.id)
+                    .unwrap_or_default();
             }
         }
 

src/aheader.rs

@@ -67,7 +67,7 @@ impl Aheader {
 
 impl fmt::Display for Aheader {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        write!(fmt, "addr={};", self.addr)?;
+        write!(fmt, "addr={};", self.addr.to_lowercase())?;
         if self.prefer_encrypt == EncryptPreference::Mutual {
             write!(fmt, " prefer-encrypt=mutual;")?;
         }
@@ -262,5 +262,16 @@ mod tests {
             )
         )
         .contains("prefer-encrypt"));
+
+        // Always lowercase the address in the header.
+        assert!(format!(
+            "{}",
+            Aheader::new(
+                "TeSt@eXaMpLe.cOm".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::Mutual
+            )
+        )
+        .contains("test@example.com"));
     }
 }

src/config.rs

@@ -5,17 +5,20 @@ use std::path::Path;
 use std::str::FromStr;
 
 use anyhow::{ensure, Context as _, Result};
+use serde::{Deserialize, Serialize};
 use strum::{EnumProperty, IntoEnumIterator};
-use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
+use strum_macros::{AsRefStr, Display, EnumIter, EnumString};
 
 use crate::blob::BlobObject;
 use crate::constants::DC_VERSION_STR;
 use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::events::EventType;
+use crate::log::LogExt;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::provider::{get_provider_by_id, Provider};
-use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
+use crate::sync::{self, Sync::*, SyncData};
+use crate::tools::{get_abs_path, improve_single_line_input};
 
 /// The available configuration keys.
 #[derive(
@@ -31,6 +34,8 @@ use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
     EnumProperty,
     PartialOrd,
     Ord,
+    Serialize,
+    Deserialize,
 )]
 #[strum(serialize_all = "snake_case")]
 pub enum Config {
@@ -338,6 +343,31 @@ pub enum Config {
     /// until `chat_id.accept()` is called.
     #[strum(props(default = "0"))]
     VerifiedOneOnOneChats,
+
+    /// Row ID of the key in the `keypairs` table
+    /// used for signatures, encryption to self and included in `Autocrypt` header.
+    KeyId,
+
+    /// Iroh secret key.
+    IrohSecretKey,
+}
+
+impl Config {
+    /// Whether the config option is synced across devices.
+    ///
+    /// This must be checked on both sides so that if there are different client versions, the
+    /// synchronisation of a particular option is either done or not done in both directions.
+    /// Moreover, receivers of a config value need to check if a key can be synced because some
+    /// settings (e.g. Avatar path) could otherwise lead to exfiltration of files from a receiver's
+    /// device if we assume an attacker to have control of a device in a multi-device setting or if
+    /// multiple users are sharing an account. Another example is `Self::SyncMsgs` itself which
+    /// mustn't be controlled by other devices.
+    pub(crate) fn is_synced(&self) -> bool {
+        matches!(
+            self,
+            Self::Displayname | Self::MdnsEnabled | Self::ShowEmails
+        )
+    }
 }
 
 impl Context {
@@ -460,6 +490,16 @@ impl Context {
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
+        self.set_config_ex(key.is_synced().into(), key, value).await
+    }
+
+    pub(crate) async fn set_config_ex(
+        &self,
+        sync: sync::Sync,
+        key: Config,
+        mut value: Option<&str>,
+    ) -> Result<()> {
+        let better_value;
         match key {
             Config::Selfavatar => {
                 self.sql
@@ -486,10 +526,11 @@ impl Context {
                 ret?
             }
             Config::Displayname => {
-                let value = value.map(improve_single_line_input);
-                self.sql
-                    .set_raw_config(key.as_ref(), value.as_deref())
-                    .await?;
+                if let Some(v) = value {
+                    better_value = improve_single_line_input(v);
+                    value = Some(&better_value);
+                }
+                self.sql.set_raw_config(key.as_ref(), value).await?;
             }
             Config::Socks5Enabled
             | Config::BccSelf
@@ -513,10 +554,33 @@ impl Context {
                 );
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
+            Config::Addr => {
+                self.sql
+                    .set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
+                    .await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
         }
+
+        if sync != Sync {
+            return Ok(());
+        }
+        self.emit_event(EventType::ConfigSynced { key });
+        let Some(val) = value else {
+            return Ok(());
+        };
+        let val = val.to_string();
+        if self
+            .add_sync_item(SyncData::Config { key, val })
+            .await
+            .log_err(self)
+            .is_err()
+        {
+            return Ok(());
+        }
+        self.send_sync_msg().await.log_err(self).ok();
         Ok(())
     }
 
@@ -571,8 +635,6 @@ impl Context {
     ///
     /// This should only be used by test code and during configure.
     pub(crate) async fn set_primary_self_addr(&self, primary_new: &str) -> Result<()> {
-        let old_addr = self.get_config(Config::ConfiguredAddr).await?;
-
         // add old primary address (if exists) to secondary addresses
         let mut secondary_addrs = self.get_all_self_addrs().await?;
         // never store a primary address also as a secondary
@@ -586,17 +648,6 @@ impl Context {
         self.set_config(Config::ConfiguredAddr, Some(primary_new))
             .await?;
 
-        if let Some(old_addr) = old_addr {
-            let old_addr = EmailAddress::new(&old_addr)?;
-            let old_keypair = crate::key::load_keypair(self, &old_addr).await?;
-
-            if let Some(mut old_keypair) = old_keypair {
-                old_keypair.addr = EmailAddress::new(primary_new)?;
-                crate::key::store_self_keypair(self, &old_keypair, crate::key::KeyPairUse::Default)
-                    .await?;
-            }
-        }
-
         Ok(())
     }
 
@@ -648,7 +699,7 @@ mod tests {
 
     use super::*;
     use crate::constants;
-    use crate::test_utils::TestContext;
+    use crate::test_utils::{sync, TestContext};
 
     #[test]
     fn test_to_string() {
@@ -662,6 +713,21 @@ mod tests {
         );
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_set_config_addr() {
+        let t = TestContext::new().await;
+
+        // Test that uppercase address get lowercased.
+        assert!(t
+            .set_config(Config::Addr, Some("Foobar@eXample.oRg"))
+            .await
+            .is_ok());
+        assert_eq!(
+            t.get_config(Config::Addr).await.unwrap().unwrap(),
+            "foobar@example.org"
+        );
+    }
+
     /// Tests that "bot" config can only be set to "0" or "1".
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_set_config_bot() {
@@ -777,4 +843,98 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync() -> Result<()> {
+        let alice0 = TestContext::new_alice().await;
+        let alice1 = TestContext::new_alice().await;
+        for a in [&alice0, &alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        let mdns_enabled = alice0.get_config_bool(Config::MdnsEnabled).await?;
+        // Alice1 has a different config value.
+        alice1
+            .set_config_bool(Config::MdnsEnabled, !mdns_enabled)
+            .await?;
+        // This changes nothing, but still sends a sync message.
+        alice0
+            .set_config_bool(Config::MdnsEnabled, mdns_enabled)
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config_bool(Config::MdnsEnabled).await?,
+            mdns_enabled
+        );
+
+        // Reset to default. Test that it's not synced because defaults may differ across client
+        // versions.
+        alice0.set_config(Config::MdnsEnabled, None).await?;
+        assert_eq!(alice0.get_config_bool(Config::MdnsEnabled).await?, true);
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::MdnsEnabled
+                    }
+                )
+            })
+            .await;
+        alice0.set_config_bool(Config::MdnsEnabled, false).await?;
+        alice0
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::MdnsEnabled
+                    }
+                )
+            })
+            .await;
+        sync(&alice0, &alice1).await;
+        assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
+        alice1
+            .evtracker
+            .get_matching(|e| {
+                matches!(
+                    e,
+                    EventType::ConfigSynced {
+                        key: Config::MdnsEnabled
+                    }
+                )
+            })
+            .await;
+
+        let show_emails = alice0.get_config_bool(Config::ShowEmails).await?;
+        alice0
+            .set_config_bool(Config::ShowEmails, !show_emails)
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config_bool(Config::ShowEmails).await?,
+            !show_emails
+        );
+
+        // `Config::SyncMsgs` mustn't be synced.
+        alice0.set_config_bool(Config::SyncMsgs, false).await?;
+        alice0.set_config_bool(Config::SyncMsgs, true).await?;
+        alice0.set_config_bool(Config::MdnsEnabled, true).await?;
+        sync(&alice0, &alice1).await;
+        assert!(alice1.get_config_bool(Config::MdnsEnabled).await?);
+
+        // Usual sync scenario.
+        alice0
+            .set_config(Config::Displayname, Some("Alice Sync"))
+            .await?;
+        sync(&alice0, &alice1).await;
+        assert_eq!(
+            alice1.get_config(Config::Displayname).await?,
+            Some("Alice Sync".to_string())
+        );
+
+        Ok(())
+    }
 }

src/configure.rs

@@ -34,6 +34,7 @@ use crate::provider::{Protocol, Socket, UsernamePattern};
 use crate::smtp::Smtp;
 use crate::socks::Socks5Config;
 use crate::stock_str;
+use crate::sync::Sync::*;
 use crate::tools::{time, EmailAddress};
 use crate::{chat, e2ee, provider};
 
@@ -132,7 +133,9 @@ async fn on_configure_completed(
             for def in config_defaults {
                 if !context.config_exists(def.key).await? {
                     info!(context, "apply config_defaults {}={}", def.key, def.value);
-                    context.set_config(def.key, Some(def.value)).await?;
+                    context
+                        .set_config_ex(Nosync, def.key, Some(def.value))
+                        .await?;
                 } else {
                     info!(
                         context,

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
+//! RFC draft: <https://www.ietf.org/archive/id/draft-bucksch-autoconfig-00.html>
+//! Archived original documentation: <https://web.archive.org/web/20210624004729/https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
 use std::io::BufRead;
 use std::str::FromStr;
 

src/configure/server_params.rs

@@ -137,20 +137,11 @@ impl ServerParams {
     }
 
     fn expand_strict_tls(self) -> Vec<ServerParams> {
-        if self.strict_tls.is_none() {
-            vec![
-                Self {
-                    strict_tls: Some(true), // Strict.
-                    ..self.clone()
-                },
-                Self {
-                    strict_tls: None, // Automatic.
-                    ..self
-                },
-            ]
-        } else {
-            vec![self]
-        }
+        vec![Self {
+            // Strict if not set by the user or provider database.
+            strict_tls: Some(self.strict_tls.unwrap_or(true)),
+            ..self
+        }]
     }
 }
 
@@ -162,31 +153,10 @@ pub(crate) fn expand_param_vector(
     domain: &str,
 ) -> Vec<ServerParams> {
     v.into_iter()
-        .map(|params| {
-            if params.socket == Socket::Plain {
-                ServerParams {
-                    // Avoid expanding plaintext configuration into configuration with and without
-                    // `strict_tls` if `strict_tls` is set to `None` as `strict_tls` is not used for
-                    // plaintext connections. Always setting it to "enabled", just in case.
-                    strict_tls: Some(true),
-                    ..params
-                }
-            } else {
-                params
-            }
-        })
         // The order of expansion is important.
         //
         // Ports are expanded the last, so they are changed the first.  Username is only changed if
         // default value (address with domain) didn't work for all available hosts and ports.
-        //
-        // Strict TLS must be expanded first, so we try all configurations with strict TLS first
-        // and only then try again without strict TLS. Otherwise we may lock to wrong hostname
-        // without strict TLS when another hostname with strict TLS is available.  For example, if
-        // both smtp.example.net and mail.example.net are running an SMTP server, but both use a
-        // certificate that is only valid for mail.example.net, we want to skip smtp.example.net
-        // and use mail.example.net with strict TLS instead of using smtp.example.net without
-        // strict TLS.
         .flat_map(|params| params.expand_strict_tls().into_iter())
         .flat_map(|params| params.expand_usernames(addr).into_iter())
         .flat_map(|params| params.expand_hostnames(domain).into_iter())
@@ -257,22 +227,6 @@ mod tests {
                     username: "foobar".to_string(),
                     strict_tls: Some(true)
                 },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Ssl,
-                    username: "foobar".to_string(),
-                    strict_tls: None,
-                },
-                ServerParams {
-                    protocol: Protocol::Smtp,
-                    hostname: "example.net".to_string(),
-                    port: 123,
-                    socket: Socket::Starttls,
-                    username: "foobar".to_string(),
-                    strict_tls: None
-                }
             ],
         );
 
@@ -284,7 +238,7 @@ mod tests {
                 port: 123,
                 socket: Socket::Plain,
                 username: "foobar".to_string(),
-                strict_tls: None,
+                strict_tls: Some(true),
             }],
             "foobar@example.net",
             "example.net",

src/constants.rs

@@ -209,6 +209,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
 
+// If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
+// chunks. This does not affect MIME's `To:` header. Can be overwritten by setting
+// `max_smtp_rcpt_to` in the provider db.
+pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: usize = 50;
+
 #[cfg(test)]
 mod tests {
     use num_traits::FromPrimitive;

src/contact.rs

@@ -26,13 +26,14 @@ use crate::constants::{Blocked, Chattype, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY}
 use crate::context::Context;
 use crate::events::EventType;
 use crate::key::{load_self_public_key, DcKey};
+use crate::log::LogExt;
 use crate::login_param::LoginParam;
 use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
-use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::peerstate::Peerstate;
 use crate::sql::{self, params_iter};
-use crate::sync::{self, Sync::*, SyncData};
+use crate::sync::{self, Sync::*};
 use crate::tools::{
     duration_to_str, get_abs_path, improve_single_line_input, strip_rtlo_characters, time,
     EmailAddress,
@@ -43,43 +44,43 @@ use crate::{chat, stock_str};
 const SEEN_RECENTLY_SECONDS: i64 = 600;
 
 /// Valid contact address.
-#[derive(Debug, Clone, Copy)]
-pub(crate) struct ContactAddress<'a>(&'a str);
+#[derive(Debug, Clone)]
+pub(crate) struct ContactAddress(String);
 
-impl Deref for ContactAddress<'_> {
+impl Deref for ContactAddress {
     type Target = str;
 
     fn deref(&self) -> &Self::Target {
-        self.0
+        &self.0
     }
 }
 
-impl AsRef<str> for ContactAddress<'_> {
+impl AsRef<str> for ContactAddress {
     fn as_ref(&self) -> &str {
-        self.0
+        &self.0
     }
 }
 
-impl fmt::Display for ContactAddress<'_> {
+impl fmt::Display for ContactAddress {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.0)
     }
 }
 
-impl<'a> ContactAddress<'a> {
+impl ContactAddress {
     /// Constructs a new contact address from string,
     /// normalizing and validating it.
-    pub fn new(s: &'a str) -> Result<Self> {
+    pub fn new(s: &str) -> Result<Self> {
         let addr = addr_normalize(s);
-        if !may_be_valid_addr(addr) {
+        if !may_be_valid_addr(&addr) {
             bail!("invalid address {:?}", s);
         }
-        Ok(Self(addr))
+        Ok(Self(addr.to_string()))
     }
 }
 
 /// Allow converting [`ContactAddress`] to an SQLite type.
-impl rusqlite::types::ToSql for ContactAddress<'_> {
+impl rusqlite::types::ToSql for ContactAddress {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         let val = rusqlite::types::Value::Text(self.0.to_string());
         let out = rusqlite::types::ToSqlOutput::Owned(val);
@@ -149,6 +150,22 @@ impl ContactId {
             .await?;
         Ok(())
     }
+
+    /// Reset gossip timestamp in all chats with this contact.
+    pub(crate) async fn regossip_keys(&self, context: &Context) -> Result<()> {
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+                 SET gossiped_timestamp=0
+                 WHERE EXISTS (SELECT 1 FROM chats_contacts
+                               WHERE chats_contacts.chat_id=chats.id
+                               AND chats_contacts.contact_id=?)",
+                (self,),
+            )
+            .await?;
+        Ok(())
+    }
 }
 
 impl fmt::Display for ContactId {
@@ -332,24 +349,6 @@ pub(crate) enum Modifier {
     Created,
 }
 
-/// Verification status of the contact.
-#[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum VerifiedStatus {
-    /// Contact is not verified.
-    Unverified = 0,
-    /// SELF has verified the fingerprint of a contact. Currently unused.
-    Verified = 1,
-    /// SELF and contact have verified their fingerprints in both directions; in the UI typically checkmarks are shown.
-    BidirectVerified = 2,
-}
-
-impl Default for VerifiedStatus {
-    fn default() -> Self {
-        Self::Unverified
-    }
-}
-
 impl Contact {
     /// Loads a single contact object from the database.
     ///
@@ -478,13 +477,22 @@ impl Contact {
     ///
     /// May result in a `#DC_EVENT_CONTACTS_CHANGED` event.
     pub async fn create(context: &Context, name: &str, addr: &str) -> Result<ContactId> {
+        Self::create_ex(context, Sync, name, addr).await
+    }
+
+    pub(crate) async fn create_ex(
+        context: &Context,
+        sync: sync::Sync,
+        name: &str,
+        addr: &str,
+    ) -> Result<ContactId> {
         let name = improve_single_line_input(name);
 
         let (name, addr) = sanitize_name_and_addr(&name, addr);
         let addr = ContactAddress::new(&addr)?;
 
         let (contact_id, sth_modified) =
-            Contact::add_or_lookup(context, &name, addr, Origin::ManuallyCreated)
+            Contact::add_or_lookup(context, &name, &addr, Origin::ManuallyCreated)
                 .await
                 .context("add_or_lookup")?;
         let blocked = Contact::is_blocked_load(context, contact_id).await?;
@@ -498,6 +506,16 @@ impl Contact {
             set_blocked(context, Nosync, contact_id, false).await?;
         }
 
+        if sync.into() {
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(addr.to_string()),
+                chat::SyncAction::Rename(name.to_string()),
+            )
+            .await
+            .log_err(context)
+            .ok();
+        }
         Ok(contact_id)
     }
 
@@ -549,7 +567,7 @@ impl Contact {
 
         let addr_normalized = addr_normalize(addr);
 
-        if context.is_self_addr(addr_normalized).await? {
+        if context.is_self_addr(&addr_normalized).await? {
             return Ok(Some(ContactId::SELF));
         }
 
@@ -599,7 +617,7 @@ impl Contact {
     pub(crate) async fn add_or_lookup(
         context: &Context,
         name: &str,
-        addr: ContactAddress<'_>,
+        addr: &ContactAddress,
         mut origin: Origin,
     ) -> Result<(ContactId, Modifier)> {
         let mut sth_modified = Modifier::None;
@@ -607,7 +625,7 @@ impl Contact {
         ensure!(!addr.is_empty(), "Can not add_or_lookup empty address");
         ensure!(origin != Origin::Unknown, "Missing valid origin");
 
-        if context.is_self_addr(&addr).await? {
+        if context.is_self_addr(addr).await? {
             return Ok((ContactId::SELF, sth_modified));
         }
 
@@ -762,7 +780,7 @@ impl Contact {
                             } else {
                                 "".to_string()
                             },
-                            addr,
+                            &addr,
                             origin,
                             if update_authname {
                                 name.to_string()
@@ -774,7 +792,7 @@ impl Contact {
 
                 sth_modified = Modifier::Created;
                 row_id = u32::try_from(transaction.last_insert_rowid())?;
-                info!(context, "added contact id={} addr={}", row_id, &addr);
+                info!(context, "Added contact id={row_id} addr={addr}.");
             }
             Ok(row_id)
         }).await?;
@@ -807,7 +825,7 @@ impl Contact {
             let name = normalize_name(&name);
             match ContactAddress::new(&addr) {
                 Ok(addr) => {
-                    match Contact::add_or_lookup(context, &name, addr, Origin::AddressBook).await {
+                    match Contact::add_or_lookup(context, &name, &addr, Origin::AddressBook).await {
                         Ok((_, modified)) => {
                             if modified != Modifier::None {
                                 modify_cnt += 1
@@ -1039,11 +1057,9 @@ impl Contact {
             let loginparam = LoginParam::load_configured_params(context).await?;
             let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
 
-            if let Some(peerstate) = peerstate.filter(|peerstate| {
-                peerstate
-                    .peek_key(PeerstateVerifiedStatus::Unverified)
-                    .is_some()
-            }) {
+            if let Some(peerstate) =
+                peerstate.filter(|peerstate| peerstate.peek_key(false).is_some())
+            {
                 let stock_message = match peerstate.prefer_encrypt {
                     EncryptPreference::Mutual => stock_str::e2e_preferred(context).await,
                     EncryptPreference::NoPreference => stock_str::e2e_available(context).await,
@@ -1058,11 +1074,11 @@ impl Contact {
                     .fingerprint()
                     .to_string();
                 let fingerprint_other_verified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::BidirectVerified)
+                    .peek_key(true)
                     .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::Unverified)
+                    .peek_key(false)
                     .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 if loginparam.addr < peerstate.addr {
@@ -1265,20 +1281,37 @@ impl Contact {
     /// otherwise use is_chat_protected().
     /// Use [Self::get_verifier_id] to display the verifier contact
     /// in the info section of the contact profile.
-    pub async fn is_verified(&self, context: &Context) -> Result<VerifiedStatus> {
+    pub async fn is_verified(&self, context: &Context) -> Result<bool> {
         // We're always sort of secured-verified as we could verify the key on this device any time with the key
         // on this device
         if self.id == ContactId::SELF {
-            return Ok(VerifiedStatus::BidirectVerified);
+            return Ok(true);
         }
 
-        if let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? {
-            if peerstate.is_using_verified_key() {
-                return Ok(VerifiedStatus::BidirectVerified);
-            }
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        let forward_verified = peerstate.is_using_verified_key();
+        let backward_verified = peerstate.is_backward_verified(context).await?;
+        Ok(forward_verified && backward_verified)
+    }
+
+    /// Returns true if we have a verified key for the contact
+    /// and it is the same as Autocrypt key.
+    /// This is enough to send messages to the contact in verified chat
+    /// and verify received messages, but not enough to display green checkmark
+    /// or add the contact to verified groups.
+    pub async fn is_forward_verified(&self, context: &Context) -> Result<bool> {
+        if self.id == ContactId::SELF {
+            return Ok(true);
         }
 
-        Ok(VerifiedStatus::Unverified)
+        let Some(peerstate) = Peerstate::from_addr(context, &self.addr).await? else {
+            return Ok(false);
+        };
+
+        Ok(peerstate.is_using_verified_key())
     }
 
     /// Returns the `ContactId` that verified the contact.
@@ -1301,7 +1334,7 @@ impl Contact {
             return Ok(None);
         };
 
-        if verifier_addr == self.addr {
+        if addr_cmp(&verifier_addr, &self.addr) {
             // Contact is directly verified via QR code.
             return Ok(Some(ContactId::SELF));
         }
@@ -1333,7 +1366,7 @@ impl Contact {
             Ok(chat_id.is_protected(context).await? == ProtectionStatus::Protected)
         } else {
             // 1:1 chat does not exist.
-            Ok(self.is_verified(context).await? == VerifiedStatus::BidirectVerified)
+            Ok(self.is_verified(context).await?)
         }
     }
 
@@ -1389,12 +1422,13 @@ pub fn may_be_valid_addr(addr: &str) -> bool {
     res.is_ok()
 }
 
-/// Returns address with whitespace trimmed and `mailto:` prefix removed.
-pub fn addr_normalize(addr: &str) -> &str {
-    let norm = addr.trim();
+/// Returns address lowercased,
+/// with whitespace trimmed and `mailto:` prefix removed.
+pub fn addr_normalize(addr: &str) -> String {
+    let norm = addr.trim().to_lowercase();
 
     if norm.starts_with("mailto:") {
-        norm.get(7..).unwrap_or(norm)
+        norm.get(7..).unwrap_or(&norm).to_string()
     } else {
         norm
     }
@@ -1483,13 +1517,14 @@ WHERE type=? AND id IN (
                 true => chat::SyncAction::Block,
                 false => chat::SyncAction::Unblock,
             };
-            context
-                .add_sync_item(SyncData::AlterChat {
-                    id: chat::SyncId::ContactAddr(contact.addr.clone()),
-                    action,
-                })
-                .await?;
-            context.send_sync_msg().await?;
+            chat::sync(
+                context,
+                chat::SyncId::ContactAddr(contact.addr.clone()),
+                action,
+            )
+            .await
+            .log_err(context)
+            .ok();
         }
     }
 
@@ -1649,8 +1684,8 @@ fn cat_fingerprint(
 
 /// Compares two email addresses, normalizing them beforehand.
 pub fn addr_cmp(addr1: &str, addr2: &str) -> bool {
-    let norm1 = addr_normalize(addr1).to_lowercase();
-    let norm2 = addr_normalize(addr2).to_lowercase();
+    let norm1 = addr_normalize(addr1);
+    let norm2 = addr_normalize(addr2);
 
     norm1 == norm2
 }
@@ -1848,10 +1883,7 @@ mod tests {
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");
         assert_eq!(addr_normalize("  hello@world.com   "), "hello@world.com");
-
-        // normalisation preserves case to allow user-defined spelling.
-        // however, case is ignored on addr_cmp()
-        assert_ne!(addr_normalize("John@Doe.com"), "john@doe.com");
+        assert_eq!(addr_normalize("John@Doe.com"), "john@doe.com");
     }
 
     #[test]
@@ -1877,7 +1909,7 @@ mod tests {
         let (id, _modified) = Contact::add_or_lookup(
             &context.ctx,
             "bob",
-            ContactAddress::new("user@example.org")?,
+            &ContactAddress::new("user@example.org")?,
             Origin::IncomingReplyTo,
         )
         .await?;
@@ -1891,12 +1923,12 @@ mod tests {
         // Search by name.
         let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         // Search by address.
         let contacts = Contact::get_all(&context.ctx, 0, Some("user")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).await?;
         assert_eq!(contacts.len(), 0);
@@ -1905,7 +1937,7 @@ mod tests {
         let (contact_bob_id, modified) = Contact::add_or_lookup(
             &context.ctx,
             "someone",
-            ContactAddress::new("user@example.org")?,
+            &ContactAddress::new("user@example.org")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -1923,7 +1955,7 @@ mod tests {
         // Search by display name (same as manually set name).
         let contacts = Contact::get_all(&context.ctx, 0, Some("someone")).await?;
         assert_eq!(contacts.len(), 1);
-        assert_eq!(contacts.get(0), Some(&id));
+        assert_eq!(contacts.first(), Some(&id));
 
         Ok(())
     }
@@ -1970,7 +2002,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bla foo",
-            ContactAddress::new("one@eins.org").unwrap(),
+            &ContactAddress::new("one@eins.org").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -1989,7 +2021,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "Real one",
-            ContactAddress::new(" one@eins.org  ").unwrap(),
+            &ContactAddress::new(" one@eins.org  ").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -2005,7 +2037,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2022,7 +2054,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "m. serious",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2037,7 +2069,7 @@ mod tests {
         let (contact_id_test, sth_modified) = Contact::add_or_lookup(
             &t,
             "schnucki",
-            ContactAddress::new("three@drei.sam").unwrap(),
+            &ContactAddress::new("three@drei.sam").unwrap(),
             Origin::ManuallyCreated,
         )
         .await
@@ -2053,7 +2085,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "",
-            ContactAddress::new("alice@w.de").unwrap(),
+            &ContactAddress::new("alice@w.de").unwrap(),
             Origin::IncomingUnknownTo,
         )
         .await
@@ -2195,7 +2227,7 @@ mod tests {
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2274,7 +2306,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob1",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2290,7 +2322,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob2",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2316,7 +2348,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "bob4",
-            ContactAddress::new("bob@example.org").unwrap(),
+            &ContactAddress::new("bob@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2345,7 +2377,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire1",
-            ContactAddress::new("claire@example.org").unwrap(),
+            &ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2361,7 +2393,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "claire2",
-            ContactAddress::new("claire@example.org").unwrap(),
+            &ContactAddress::new("claire@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2386,7 +2418,7 @@ mod tests {
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2398,7 +2430,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Not Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownTo,
         )
         .await?;
@@ -2411,7 +2443,7 @@ mod tests {
         let (contact_id_same, sth_modified) = Contact::add_or_lookup(
             &t,
             "Bob",
-            ContactAddress::new("bob@example.org")?,
+            &ContactAddress::new("bob@example.org")?,
             Origin::IncomingUnknownFrom,
         )
         .await?;
@@ -2440,7 +2472,7 @@ mod tests {
         Contact::add_or_lookup(
             &t,
             "dave2",
-            ContactAddress::new("dave@example.org").unwrap(),
+            &ContactAddress::new("dave@example.org").unwrap(),
             Origin::IncomingUnknownFrom,
         )
         .await
@@ -2561,7 +2593,7 @@ mod tests {
         let (contact_bob_id, _modified) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2724,7 +2756,7 @@ CCCB 5AA9 F6E1 141C 9431
         let (contact_id, _) = Contact::add_or_lookup(
             &alice,
             "Bob",
-            ContactAddress::new("bob@example.net")?,
+            &ContactAddress::new("bob@example.net")?,
             Origin::ManuallyCreated,
         )
         .await?;
@@ -2795,4 +2827,33 @@ Hi."#;
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_sync_create() -> Result<()> {
+        let alice0 = &TestContext::new_alice().await;
+        let alice1 = &TestContext::new_alice().await;
+        for a in [alice0, alice1] {
+            a.set_config_bool(Config::SyncMsgs, true).await?;
+        }
+
+        Contact::create(alice0, "Bob", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let a1b_contact_id =
+            Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+                .await?
+                .unwrap();
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob");
+
+        Contact::create(alice0, "Bob Renamed", "bob@example.net").await?;
+        test_utils::sync(alice0, alice1).await;
+        let id = Contact::lookup_id_by_addr(alice1, "bob@example.net", Origin::ManuallyCreated)
+            .await?
+            .unwrap();
+        assert_eq!(id, a1b_contact_id);
+        let a1b_contact = Contact::get_by_id(alice1, a1b_contact_id).await?;
+        assert_eq!(a1b_contact.name, "Bob Renamed");
+
+        Ok(())
+    }
 }

src/context.rs

@@ -10,6 +10,8 @@ use std::time::{Duration, Instant, SystemTime};
 
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
+use iroh_gossip::net::Gossip;
+use iroh_net::MagicEndpoint;
 use ratelimit::Ratelimit;
 use tokio::sync::{Mutex, Notify, RwLock};
 
@@ -244,6 +246,12 @@ pub struct InnerContext {
     /// Standard RwLock instead of [`tokio::sync::RwLock`] is used
     /// because the lock is used from synchronous [`Context::emit_event`].
     pub(crate) debug_logging: std::sync::RwLock<Option<DebugLogging>>,
+
+    /// [MagicEndpoint] needed for iroh peer channels.
+    pub(crate) endpoint: Mutex<Option<MagicEndpoint>>,
+
+    /// [Gossip] needed for iroh peer channels.
+    pub(crate) gossip: Mutex<Option<Gossip>>,
 }
 
 /// The state of ongoing process.
@@ -388,6 +396,8 @@ impl Context {
             last_full_folder_scan: Mutex::new(None),
             last_error: std::sync::RwLock::new("".to_string()),
             debug_logging: std::sync::RwLock::new(None),
+            endpoint: Mutex::new(None),
+            gossip: Mutex::new(None),
         };
 
         let ctx = Context {
@@ -413,14 +423,21 @@ impl Context {
                 .is_some()
             {
                 let mut lock = self.ratelimit.write().await;
-                *lock = Ratelimit::new(Duration::new(40, 0), 6.0);
+                // Allow at least 1 message every second + a burst of 3.
+                *lock = Ratelimit::new(Duration::new(3, 0), 3.0);
             }
         }
+
+        if let Err(e) = self.create_gossip().await {
+            warn!(self, "{e}");
+        }
         self.scheduler.start(self.clone()).await;
     }
 
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
+        self.endpoint.lock().await.take();
+        self.gossip.lock().await.take();
         self.scheduler.stop(self).await;
     }
 
@@ -432,6 +449,9 @@ impl Context {
 
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
+        if let Some(ref mut endpoint) = *self.endpoint.lock().await {
+            endpoint.network_change().await;
+        }
         self.scheduler.maybe_network().await;
     }
 
@@ -1317,6 +1337,8 @@ mod tests {
             "socks5_port",
             "socks5_user",
             "socks5_password",
+            "key_id",
+            "iroh_secret_key",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();
@@ -1368,7 +1390,7 @@ mod tests {
         assert_eq!(res.len(), 2);
 
         // Message added later is returned first.
-        assert_eq!(res.get(0), Some(&msg2.id));
+        assert_eq!(res.first(), Some(&msg2.id));
         assert_eq!(res.get(1), Some(&msg1.id));
 
         // Global search with longer text does not find any message.
@@ -1585,7 +1607,7 @@ mod tests {
 
         let bob_next_msg_ids = bob.get_next_msgs().await?;
         assert_eq!(bob_next_msg_ids.len(), 1);
-        assert_eq!(bob_next_msg_ids.get(0), Some(&received_msg.id));
+        assert_eq!(bob_next_msg_ids.first(), Some(&received_msg.id));
 
         bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
             .await?;
@@ -1594,7 +1616,7 @@ mod tests {
         // Next messages include self-sent messages.
         let alice_next_msg_ids = alice.get_next_msgs().await?;
         assert_eq!(alice_next_msg_ids.len(), 1);
-        assert_eq!(alice_next_msg_ids.get(0), Some(&sent_msg.sender_msg_id));
+        assert_eq!(alice_next_msg_ids.first(), Some(&sent_msg.sender_msg_id));
 
         alice
             .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())
@@ -1603,4 +1625,15 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_keypair_saving() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let key = alice.get_or_generate_iroh_keypair().await?;
+        let loaded_key = alice.get_or_generate_iroh_keypair().await?;
+        assert_eq!(key.to_bytes(), loaded_key.to_bytes());
+
+        Ok(())
+    }
 }

src/debug_logging.rs

@@ -54,7 +54,7 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
         match context
             .write_status_update_inner(
                 &msg_id,
-                StatusUpdateItem {
+                &StatusUpdateItem {
                     payload: json!({
                         "event": event,
                         "time": time,
@@ -62,6 +62,8 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                     info: None,
                     summary: None,
                     document: None,
+                    uid: None,
+                    gossip_topic: None,
                 },
             )
             .await
@@ -70,10 +72,15 @@ pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLo
                 eprintln!("Can't log event to webxdc status update: {err:#}");
             }
             Ok(serial) => {
-                context.emit_event(EventType::WebxdcStatusUpdate {
-                    msg_id,
-                    status_update_serial: serial,
-                });
+                if let Some(serial) = serial {
+                    context.emit_event(EventType::WebxdcStatusUpdate {
+                        msg_id,
+                        status_update_serial: serial,
+                    });
+                } else {
+                    // This should not happen as the update has no `uid`.
+                    error!(context, "Debug logging update is not created.");
+                };
             }
         }
     }

src/decrypt.rs

@@ -23,32 +23,14 @@ use crate::pgp;
 ///
 /// If the message is wrongly signed, HashSet will be empty.
 pub fn try_decrypt(
-    context: &Context,
     mail: &ParsedMail<'_>,
     private_keyring: &[SignedSecretKey],
     public_keyring_for_validate: &[SignedPublicKey],
 ) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let encrypted_data_part = match {
-        let mime = get_autocrypt_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected Autocrypt-mime message.");
-        }
-        mime
-    }
-    .or_else(|| {
-        let mime = get_mixed_up_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected mixed-up mime message.");
-        }
-        mime
-    })
-    .or_else(|| {
-        let mime = get_attachment_mime(mail);
-        if mime.is_some() {
-            info!(context, "Detected attached Autocrypt-mime message.");
-        }
-        mime
-    }) {
+    let encrypted_data_part = match get_autocrypt_mime(mail)
+        .or_else(|| get_mixed_up_mime(mail))
+        .or_else(|| get_attachment_mime(mail))
+    {
         None => return Ok(None),
         Some(res) => res,
     };

src/download.rs

@@ -146,29 +146,19 @@ pub(crate) async fn download_msg(context: &Context, msg_id: MsgId, imap: &mut Im
         )
         .await?;
 
-    if let Some((server_uid, server_folder)) = row {
-        match imap
-            .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
-            .await
-        {
-            ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                msg.id
-                    .update_download_state(context, DownloadState::Failure)
-                    .await?;
-                Err(anyhow!("Call download_full() again to try over."))
-            }
-            ImapActionResult::Success => {
-                // update_download_state() not needed as receive_imf() already
-                // set the state and emitted the event.
-                Ok(())
-            }
-        }
-    } else {
+    let Some((server_uid, server_folder)) = row else {
         // No IMAP record found, we don't know the UID and folder.
-        msg.id
-            .update_download_state(context, DownloadState::Failure)
-            .await?;
-        Err(anyhow!("Call download_full() again to try over."))
+        return Err(anyhow!("Call download_full() again to try over."));
+    };
+
+    match imap
+        .fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
+        .await
+    {
+        ImapActionResult::RetryLater | ImapActionResult::Failed => {
+            Err(anyhow!("Call download_full() again to try over."))
+        }
+        ImapActionResult::Success => Ok(()),
     }
 }
 

src/e2ee.rs

@@ -7,7 +7,7 @@ use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
 use crate::context::Context;
 use crate::key::{load_self_public_key, load_self_secret_key, SignedPublicKey};
-use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
+use crate::peerstate::Peerstate;
 use crate::pgp;
 
 #[derive(Debug)]
@@ -52,7 +52,7 @@ impl EncryptHelper {
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
-        peerstates: &[(Option<Peerstate>, &str)],
+        peerstates: &[(Option<Peerstate>, String)],
     ) -> Result<bool> {
         let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
             1
@@ -62,21 +62,19 @@ impl EncryptHelper {
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    info!(
-                        context,
-                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
-                    );
+                    let prefer_encrypt = peerstate.prefer_encrypt;
+                    info!(context, "Peerstate for {addr:?} is {prefer_encrypt}.");
                     match peerstate.prefer_encrypt {
                         EncryptPreference::NoPreference | EncryptPreference::Reset => {}
                         EncryptPreference::Mutual => prefer_encrypt_count += 1,
                     };
                 }
                 None => {
-                    let msg = format!("peerstate for {addr:?} missing, cannot encrypt");
+                    let msg = format!("Peerstate for {addr:?} missing, cannot encrypt");
                     if e2ee_guaranteed {
-                        return Err(format_err!("{}", msg));
+                        return Err(format_err!("{msg}"));
                     } else {
-                        info!(context, "{}", msg);
+                        info!(context, "{msg}.");
                         return Ok(false);
                     }
                 }
@@ -94,9 +92,9 @@ impl EncryptHelper {
     pub async fn encrypt(
         self,
         context: &Context,
-        min_verified: PeerstateVerifiedStatus,
+        verified: bool,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate>, &str)>,
+        peerstates: Vec<(Option<Peerstate>, String)>,
     ) -> Result<String> {
         let mut keyring: Vec<SignedPublicKey> = Vec::new();
 
@@ -107,7 +105,7 @@ impl EncryptHelper {
             .filter_map(|(state, addr)| state.clone().map(|s| (s, addr)))
         {
             let key = peerstate
-                .take_key(min_verified)
+                .take_key(verified)
                 .with_context(|| format!("proper enc-key for {addr} missing, cannot encrypt"))?;
             keyring.push(key);
             verifier_addresses.push(addr);
@@ -118,8 +116,8 @@ impl EncryptHelper {
 
         // Encrypt to secondary verified keys
         // if we also encrypt to the introducer ("verifier") of the key.
-        if min_verified == PeerstateVerifiedStatus::BidirectVerified {
-            for (peerstate, _addr) in peerstates {
+        if verified {
+            for (peerstate, _addr) in &peerstates {
                 if let Some(peerstate) = peerstate {
                     if let (Some(key), Some(verifier)) = (
                         peerstate.secondary_verified_key.as_ref(),
@@ -171,11 +169,10 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chat;
     use crate::key::DcKey;
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
-    use crate::test_utils::{bob_keypair, TestContext};
+    use crate::test_utils::{bob_keypair, TestContext, TestContextManager};
 
     mod ensure_secret_key_exists {
         use super::*;
@@ -219,37 +216,35 @@ Sent with my Delta Chat Messenger: https://delta.chat";
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
 
         let chat_alice = alice.create_chat(&bob).await.id;
         let chat_bob = bob.create_chat(&alice).await.id;
 
         // Alice sends unencrypted message to Bob
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
         // Bob receives unencrypted message from Alice
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
 
-        // Parsing a message is enough to update peerstate
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
         assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
 
-        // Bob sends encrypted message to Alice
+        // Bob sends empty encrypted message to Alice
         let mut msg = Message::new(Viewtype::Text);
-        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
-        let sent = bob.pop_sent_msg().await;
+        let sent = bob.send_msg(chat_bob, &mut msg).await;
 
-        // Alice receives encrypted message from Bob
-        let msg = alice.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        // Alice receives an empty encrypted message from Bob.
+        // This is also a regression test for previously existing bug
+        // that resulted in no padlock on encrypted empty messages.
+        let msg = alice.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
 
         let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
             .await?
@@ -261,12 +256,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends encrypted message without Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -275,12 +268,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         // Alice sends plaintext message with Autocrypt header.
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -290,12 +281,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let mut msg = Message::new(Viewtype::Text);
         msg.force_plaintext();
         msg.param.set_int(Param::SkipAutocrypt, 1);
-        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
-        let sent = alice.pop_sent_msg().await;
+        let sent = alice.send_msg(chat_alice, &mut msg).await;
 
-        let msg = bob.parse_msg(&sent).await;
-        assert!(!msg.was_encrypted());
+        let msg = bob.recv_msg(&sent).await;
+        assert!(!msg.get_showpadlock());
         let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.org")
             .await?
             .expect("no peerstate found in the database");
@@ -304,7 +293,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         Ok(())
     }
 
-    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, &'static str)> {
+    fn new_peerstates(prefer_encrypt: EncryptPreference) -> Vec<(Option<Peerstate>, String)> {
         let addr = "bob@foo.bar";
         let pub_key = bob_keypair().public;
         let peerstate = Peerstate {
@@ -323,9 +312,10 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             secondary_verified_key: None,
             secondary_verified_key_fingerprint: None,
             secondary_verifier: None,
+            backward_verified_key_id: None,
             fingerprint_changed: false,
         };
-        vec![(Some(peerstate), addr)]
+        vec![(Some(peerstate), addr.to_string())]
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -350,7 +340,7 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
 
         // test with missing peerstate
-        let ps = vec![(None, "bob@foo.bar")];
+        let ps = vec![(None, "bob@foo.bar".to_string())];
         assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
         assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }

src/events/payload.rs

@@ -4,6 +4,7 @@ use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 use crate::chat::ChatId;
+use crate::config::Config;
 use crate::contact::ContactId;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
@@ -261,8 +262,17 @@ pub enum EventType {
     ConnectivityChanged,
 
     /// The user's avatar changed.
+    /// Deprecated by `ConfigSynced`.
     SelfavatarChanged,
 
+    /// A multi-device synced config value changed. Maybe the app needs to refresh smth. For
+    /// uniformity this is emitted on the source device too. The value isn't here, otherwise it
+    /// would be logged which might not be good for privacy.
+    ConfigSynced {
+        /// Configuration key.
+        key: Config,
+    },
+
     /// Webxdc status update received.
     WebxdcStatusUpdate {
         /// Message ID.

src/headerdef.rs

@@ -38,6 +38,9 @@ pub enum HeaderDef {
     /// Mailing list ID defined in [RFC 2919](https://tools.ietf.org/html/rfc2919).
     ListId,
     ListPost,
+
+    /// List-Help header defined in [RFC 2369](https://datatracker.ietf.org/doc/html/rfc2369).
+    ListHelp,
     References,
 
     /// In-Reply-To header containing Message-ID of the parent message.
@@ -85,6 +88,9 @@ pub enum HeaderDef {
     /// See <https://datatracker.ietf.org/doc/html/rfc8601>
     AuthenticationResults,
 
+    /// Public key to join gossip network.
+    IrohPublicGossip,
+
     #[cfg(test)]
     TestHeader,
 }

src/imap.rs

@@ -9,6 +9,7 @@ use std::{
     collections::{BTreeMap, BTreeSet, HashMap},
     iter::Peekable,
     mem::take,
+    time::Duration,
 };
 
 use anyhow::{bail, format_err, Context as _, Result};
@@ -16,6 +17,8 @@ use async_channel::Receiver;
 use async_imap::types::{Fetch, Flag, Name, NameAttribute, UnsolicitedResponse};
 use futures::{StreamExt, TryStreamExt};
 use num_traits::FromPrimitive;
+use ratelimit::Ratelimit;
+use tokio::sync::RwLock;
 
 use crate::chat::{self, ChatId, ChatIdBlocked};
 use crate::config::Config;
@@ -38,7 +41,7 @@ use crate::scheduler::connectivity::ConnectivityStore;
 use crate::socks::Socks5Config;
 use crate::sql;
 use crate::stock_str;
-use crate::tools::create_id;
+use crate::tools::{create_id, duration_to_str};
 
 pub(crate) mod capabilities;
 mod client;
@@ -91,6 +94,21 @@ pub struct Imap {
     login_failed_once: bool,
 
     pub(crate) connectivity: ConnectivityStore,
+
+    /// Rate limit for IMAP connection usage attempts.
+    ///
+    /// Rate limit is checked before connecting
+    /// and updated right before login attempt.
+    /// It does not limit the number of connection attempts
+    /// if the network is bad as only successful connections are counted.
+    ///
+    /// Main purpose of this rate limit is
+    /// to prevent busy loop in case
+    /// connection gets dropped over and over due to IMAP bug,
+    /// e.g. the server returning invalid response to SELECT command
+    /// immediately after logging in or returning an error in response to LOGIN command
+    /// due to internal server error.
+    ratelimit: RwLock<Ratelimit>,
 }
 
 #[derive(Debug)]
@@ -252,6 +270,8 @@ impl Imap {
             session: None,
             login_failed_once: false,
             connectivity: Default::default(),
+            // 1 login per minute + a burst of 2.
+            ratelimit: RwLock::new(Ratelimit::new(Duration::new(120, 0), 2.0)),
         };
 
         Ok(imap)
@@ -301,7 +321,20 @@ impl Imap {
 
         self.connectivity.set_connecting(context).await;
 
-        let oauth2 = self.config.lp.oauth2;
+        // Check rate limit before trying to connect
+        // to avoid connecting and not using the connection
+        // in case we are currently ratelimited.
+        // Otherwise connection may become unusable due to NAT forgetting about it
+        // before we attempt to actually login.
+        let ratelimit_duration = self.ratelimit.read().await.until_can_send();
+        if !ratelimit_duration.is_zero() {
+            warn!(
+                context,
+                "IMAP got rate limited, waiting for {} until can connect",
+                duration_to_str(ratelimit_duration),
+            );
+            tokio::time::sleep(ratelimit_duration).await;
+        }
 
         info!(context, "Connecting to IMAP server");
         let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
@@ -353,11 +386,13 @@ impl Imap {
                 Client::connect_secure(context, imap_server, imap_port, config.strict_tls).await
             }
         };
-
         let client = connection_res?;
+        self.ratelimit.write().await.send();
+
         let config = &self.config;
         let imap_user: &str = config.lp.user.as_ref();
         let imap_pw: &str = config.lp.password.as_ref();
+        let oauth2 = self.config.lp.oauth2;
 
         let login_res = if oauth2 {
             info!(context, "Logging into IMAP server with OAuth 2");
@@ -572,9 +607,6 @@ impl Imap {
     /// When selecting a folder for the first time, sets the uid_next to the current
     /// mailbox.uid_next so that no old emails are fetched.
     ///
-    /// Makes sure that UIDNEXT is known for `selected_mailbox`
-    /// and errors out if UIDNEXT cannot be determined.
-    ///
     /// Returns Result<new_emails> (i.e. whether new emails arrived),
     /// if in doubt, returns new_emails=true so emails are fetched.
     pub(crate) async fn select_with_uidvalidity(
@@ -592,11 +624,18 @@ impl Imap {
             .as_mut()
             .with_context(|| format!("No mailbox selected, folder: {folder}"))?;
 
+        let old_uid_validity = get_uidvalidity(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
+        let old_uid_next = get_uid_next(context, folder)
+            .await
+            .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+
         let new_uid_validity = mailbox
             .uid_validity
             .with_context(|| format!("No UIDVALIDITY for folder {folder}"))?;
         let new_uid_next = if let Some(uid_next) = mailbox.uid_next {
-            uid_next
+            Some(uid_next)
         } else {
             warn!(
                 context,
@@ -619,20 +658,17 @@ impl Imap {
                 .inner
                 .status(folder, "(UIDNEXT)")
                 .await
-                .context("STATUS (UIDNEXT) error for {folder:?}")?;
+                .with_context(|| format!("STATUS (UIDNEXT) error for {folder:?}"))?;
 
-            status
-                .uid_next
-                .context("STATUS {folder} (UIDNEXT) did not return UIDNEXT")?
+            if status.uid_next.is_none() {
+                // This happens with mail.163.com as of 2023-11-26.
+                // It does not return UIDNEXT on SELECT and returns invalid
+                // `* STATUS "INBOX" ()` response on explicit request for UIDNEXT.
+                warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT.");
+            }
+            status.uid_next
         };
-        mailbox.uid_next = Some(new_uid_next);
-
-        let old_uid_validity = get_uidvalidity(context, folder)
-            .await
-            .with_context(|| format!("failed to get old UID validity for folder {folder}"))?;
-        let old_uid_next = get_uid_next(context, folder)
-            .await
-            .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
+        mailbox.uid_next = new_uid_next;
 
         if new_uid_validity == old_uid_validity {
             let new_emails = if newly_selected == NewlySelected::No {
@@ -641,7 +677,7 @@ impl Imap {
                 // the caller tries to fetch new messages (we could of course run a SELECT command now, but trying to fetch
                 // new messages is only one command, just as a SELECT command)
                 true
-            } else {
+            } else if let Some(new_uid_next) = new_uid_next {
                 if new_uid_next < old_uid_next {
                     warn!(
                         context,
@@ -651,7 +687,11 @@ impl Imap {
                     context.schedule_resync().await?;
                 }
                 new_uid_next != old_uid_next // If UIDNEXT changed, there are new emails
+            } else {
+                // We have no UIDNEXT and if in doubt, return true.
+                true
             };
+
             return Ok(new_emails);
         }
 
@@ -660,6 +700,7 @@ impl Imap {
 
         // ==============  uid_validity has changed or is being set the first time.  ==============
 
+        let new_uid_next = new_uid_next.unwrap_or_default();
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
 
@@ -876,11 +917,7 @@ impl Imap {
             .as_ref()
             .with_context(|| format!("Expected {folder:?} to be selected"))?
             .uid_next
-            .with_context(|| {
-                format!(
-                    "Expected UIDNEXT to be determined for {folder:?} by select_with_uidvalidity"
-                )
-            })?;
+            .unwrap_or_default();
         let new_uid_next = max(
             max(largest_uid_fetched, largest_uid_skipped.unwrap_or(0)) + 1,
             mailbox_uid_next,
@@ -957,7 +994,7 @@ impl Imap {
         self.prepare(context).await?;
 
         let all_folders = self
-            .list_folders(context)
+            .list_folders()
             .await
             .context("listing folders for resync")?;
         for folder in all_folders {
@@ -1451,13 +1488,8 @@ impl Imap {
                             break;
                         };
 
-                    let next_fetch_response = match next_fetch_response {
-                        Ok(next_fetch_response) => next_fetch_response,
-                        Err(err) => {
-                            warn!(context, "Failed to process IMAP FETCH result: {}.", err);
-                            continue;
-                        }
-                    };
+                    let next_fetch_response =
+                        next_fetch_response.context("Failed to process IMAP FETCH result")?;
 
                     if let Some(next_uid) = next_fetch_response.uid {
                         if next_uid == request_uid {
@@ -2519,7 +2551,7 @@ async fn add_all_recipients_as_contacts(
         let (_, modified) = Contact::add_or_lookup(
             context,
             &display_name_normalized,
-            recipient_addr,
+            &recipient_addr,
             Origin::OutgoingTo,
         )
         .await?;

src/imap/client.rs

@@ -17,8 +17,8 @@ use crate::net::tls::wrap_tls;
 use crate::socks::Socks5Config;
 use fast_socks5::client::Socks5Stream;
 
-/// IMAP write and read timeout.
-pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(30);
+/// IMAP connection, write and read timeout.
+pub(crate) const IMAP_TIMEOUT: Duration = Duration::from_secs(60);
 
 #[derive(Debug)]
 pub(crate) struct Client {

src/imap/idle.rs

@@ -9,10 +9,16 @@ use super::session::Session;
 use super::Imap;
 use crate::config::Config;
 use crate::context::Context;
-use crate::imap::{client::IMAP_TIMEOUT, get_uid_next, FolderMeaning};
+use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
 use crate::log::LogExt;
 
-const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
+/// Timeout after which IDLE is finished
+/// if there are no responses from the server.
+///
+/// If `* OK Still here` keepalives are sent more frequently
+/// than this duration, timeout should never be triggered.
+/// For example, Dovecot sends keepalives every 2 minutes by default.
+const IDLE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
 
 impl Session {
     pub async fn idle(
@@ -29,29 +35,6 @@ impl Session {
             return Ok(self);
         }
 
-        // Despite checking for unsolicited EXISTS above,
-        // we may have missed EXISTS if the message was
-        // received when the folder was not selected.
-        let status = self
-            .status(folder, "(UIDNEXT)")
-            .await
-            .context("STATUS (UIDNEXT) error for {folder:?}")?;
-        if let Some(uid_next) = status.uid_next {
-            let expected_uid_next = get_uid_next(context, folder)
-                .await
-                .with_context(|| format!("failed to get old UID NEXT for folder {folder}"))?;
-            if uid_next > expected_uid_next {
-                info!(
-                    context,
-                    "Skipping IDLE on {folder:?} because UIDNEXT {uid_next}>{expected_uid_next} indicates there are new messages."
-                );
-                return Ok(self);
-            }
-        } else {
-            warn!(context, "STATUS {folder} (UIDNEXT) did not return UIDNEXT");
-            // Go to IDLE anyway if STATUS is broken.
-        }
-
         if let Ok(()) = idle_interrupt_receiver.try_recv() {
             info!(context, "skip idle, got interrupt");
             return Ok(self);

src/imap/scan_folders.rs

@@ -1,7 +1,7 @@
 use std::{collections::BTreeMap, time::Instant};
 
 use anyhow::{Context as _, Result};
-use futures::stream::StreamExt;
+use futures::TryStreamExt;
 
 use super::{get_folder_meaning_by_attrs, get_folder_meaning_by_name};
 use crate::config::Config;
@@ -27,7 +27,7 @@ impl Imap {
         info!(context, "Starting full folder scan");
 
         self.prepare(context).await?;
-        let folders = self.list_folders(context).await?;
+        let folders = self.list_folders().await?;
         let watched_folders = get_watched_folders(context).await?;
 
         let mut folder_configs = BTreeMap::new();
@@ -98,21 +98,15 @@ impl Imap {
     }
 
     /// Returns the names of all folders on the IMAP server.
-    pub async fn list_folders(
-        self: &mut Imap,
-        context: &Context,
-    ) -> Result<Vec<async_imap::types::Name>> {
+    pub async fn list_folders(self: &mut Imap) -> Result<Vec<async_imap::types::Name>> {
         let session = self.session.as_mut();
         let session = session.context("No IMAP connection")?;
         let list = session
             .list(Some(""), Some("*"))
             .await?
-            .filter_map(|f| async {
-                f.context("list_folders() can't get folder")
-                    .log_err(context)
-                    .ok()
-            });
-        Ok(list.collect().await)
+            .try_collect()
+            .await?;
+        Ok(list)
     }
 }
 

src/imex.rs

@@ -375,7 +375,15 @@ async fn imex_inner(
     path: &Path,
     passphrase: Option<String>,
 ) -> Result<()> {
-    info!(context, "Import/export dir: {}", path.display());
+    info!(
+        context,
+        "{} path: {}",
+        match what {
+            ImexMode::ExportSelfKeys | ImexMode::ExportBackup => "Export",
+            ImexMode::ImportSelfKeys | ImexMode::ImportBackup => "Import",
+        },
+        path.display()
+    );
     ensure!(context.sql.is_open().await, "Database not opened.");
     context.emit_event(EventType::ImexProgress(10));
 
@@ -670,7 +678,7 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
     let keys = context
         .sql
         .query_map(
-            "SELECT id, public_key, private_key, is_default FROM keypairs;",
+            "SELECT id, public_key, private_key, id=(SELECT value FROM config WHERE keyname='key_id') FROM keypairs;",
             (),
             |row| {
                 let id = row.get(0)?;
@@ -808,6 +816,7 @@ mod tests {
     use tokio::task;
 
     use super::*;
+    use crate::key;
     use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
     use crate::stock_str::StockMessage;
     use crate::test_utils::{alice_keypair, TestContext};
@@ -920,6 +929,37 @@ mod tests {
         }
     }
 
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_import_second_key() -> Result<()> {
+        let alice = &TestContext::new_alice().await;
+        let chat = alice.create_chat(alice).await;
+        let sent = alice.send_text(chat.id, "Encrypted with old key").await;
+        let export_dir = tempfile::tempdir().unwrap();
+
+        let alice = &TestContext::new().await;
+        alice.configure_addr("alice@example.org").await;
+        imex(alice, ImexMode::ExportSelfKeys, export_dir.path(), None).await?;
+
+        let alice = &TestContext::new_alice().await;
+        let old_key = key::load_self_secret_key(alice).await?;
+
+        imex(alice, ImexMode::ImportSelfKeys, export_dir.path(), None).await?;
+
+        let new_key = key::load_self_secret_key(alice).await?;
+        assert_ne!(new_key, old_key);
+        assert_eq!(
+            key::load_self_secret_keyring(alice).await?,
+            vec![new_key, old_key]
+        );
+
+        let msg = alice.recv_msg(&sent).await;
+        assert!(msg.get_showpadlock());
+        assert_eq!(msg.chat_id, alice.get_self_chat().await.id);
+        assert_eq!(msg.get_text(), "Encrypted with old key");
+
+        Ok(())
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_backup() -> Result<()> {
         for set_verified_oneonone_chats in [true, false] {

src/imex/transfer.rs

@@ -638,7 +638,7 @@ mod tests {
         let self_chat = ctx1.get_self_chat().await;
         let msgs = get_chat_msgs(&ctx1, self_chat.id).await.unwrap();
         assert_eq!(msgs.len(), 2);
-        let msgid = match msgs.get(0).unwrap() {
+        let msgid = match msgs.first().unwrap() {
             ChatItem::Message { msg_id } => msg_id,
             _ => panic!("wrong chat item"),
         };

src/key.rs

@@ -16,8 +16,9 @@ use tokio::runtime::Handle;
 use crate::config::Config;
 use crate::constants::KeyGenType;
 use crate::context::Context;
+use crate::log::LogExt;
 use crate::pgp::KeyPair;
-use crate::tools::{time, EmailAddress};
+use crate::tools::EmailAddress;
 
 /// Convenience trait for working with keys.
 ///
@@ -81,10 +82,9 @@ pub(crate) async fn load_self_public_key(context: &Context) -> Result<SignedPubl
     match context
         .sql
         .query_row_optional(
-            r#"SELECT public_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT public_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -105,10 +105,9 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
     match context
         .sql
         .query_row_optional(
-            r#"SELECT private_key
-               FROM keypairs
-               WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1"#,
+            "SELECT private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
             (),
             |row| {
                 let bytes: Vec<u8> = row.get(0)?;
@@ -125,6 +124,24 @@ pub(crate) async fn load_self_secret_key(context: &Context) -> Result<SignedSecr
     }
 }
 
+pub(crate) async fn load_self_secret_keyring(context: &Context) -> Result<Vec<SignedSecretKey>> {
+    let keys = context
+        .sql
+        .query_map(
+            r#"SELECT private_key
+               FROM keypairs
+               ORDER BY id=(SELECT value FROM config WHERE keyname='key_id') DESC"#,
+            (),
+            |row| row.get::<_, Vec<u8>>(0),
+            |keys| keys.collect::<Result<Vec<_>, _>>().map_err(Into::into),
+        )
+        .await?
+        .into_iter()
+        .filter_map(|bytes| SignedSecretKey::from_slice(&bytes).log_err(context).ok())
+        .collect();
+    Ok(keys)
+}
+
 impl DcKey for SignedPublicKey {
     fn to_asc(&self, header: Option<(&str, &str)>) -> String {
         // Not using .to_armored_string() to make clear *why* it is
@@ -213,13 +230,10 @@ pub(crate) async fn load_keypair(
     let res = context
         .sql
         .query_row_optional(
-            r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-            (addr,),
+            "SELECT public_key, private_key
+             FROM keypairs
+             WHERE id=(SELECT value FROM config WHERE keyname='key_id')",
+            (),
             |row| {
                 let pub_bytes: Vec<u8> = row.get(0)?;
                 let sec_bytes: Vec<u8> = row.get(1)?;
@@ -268,42 +282,44 @@ pub async fn store_self_keypair(
     keypair: &KeyPair,
     default: KeyPairUse,
 ) -> Result<()> {
-    context
+    let mut config_cache_lock = context.sql.config_cache.write().await;
+    let new_key_id = context
         .sql
         .transaction(|transaction| {
             let public_key = DcKey::to_bytes(&keypair.public);
             let secret_key = DcKey::to_bytes(&keypair.secret);
-            transaction
-                .execute(
-                    "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-                    (&public_key, &secret_key),
-                )
-                .context("failed to remove old use of key")?;
-            if default == KeyPairUse::Default {
-                transaction
-                    .execute("UPDATE keypairs SET is_default=0;", ())
-                    .context("failed to clear default")?;
-            }
+
             let is_default = match default {
-                KeyPairUse::Default => i32::from(true),
-                KeyPairUse::ReadOnly => i32::from(false),
+                KeyPairUse::Default => true,
+                KeyPairUse::ReadOnly => false,
             };
 
-            let addr = keypair.addr.to_string();
-            let t = time();
-
             transaction
                 .execute(
-                    "INSERT INTO keypairs (addr, is_default, public_key, private_key, created)
-                VALUES (?,?,?,?,?);",
-                    (addr, is_default, &public_key, &secret_key, t),
+                    "INSERT OR REPLACE INTO keypairs (public_key, private_key)
+                     VALUES (?,?)",
+                    (&public_key, &secret_key),
                 )
-                .context("failed to insert keypair")?;
+                .context("Failed to insert keypair")?;
 
-            Ok(())
+            if is_default {
+                let new_key_id = transaction.last_insert_rowid();
+                transaction.execute(
+                    "INSERT OR REPLACE INTO config (keyname, value) VALUES ('key_id', ?)",
+                    (new_key_id,),
+                )?;
+                Ok(Some(new_key_id))
+            } else {
+                Ok(None)
+            }
         })
         .await?;
 
+    if let Some(new_key_id) = new_key_id {
+        // Update config cache if transaction succeeded and changed current default key.
+        config_cache_lock.insert("key_id".to_string(), Some(new_key_id.to_string()));
+    }
+
     Ok(())
 }
 

src/lib.rs

@@ -105,6 +105,7 @@ pub mod receive_imf;
 pub mod tools;
 
 pub mod accounts;
+pub mod peer_channels;
 pub mod reaction;
 
 /// If set IMAP/incoming and SMTP/outgoing MIME messages will be printed.

src/location.rs

@@ -954,7 +954,7 @@ Content-Disposition: attachment; filename="location.kml"
         assert!(msg.chat_id == bob_chat_id);
         assert_eq!(msg.msg_ids.len(), 1);
 
-        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.get(0).unwrap()).await?;
+        let bob_msg = Message::load_from_db(&bob, *msg.msg_ids.first().unwrap()).await?;
         assert_eq!(bob_msg.chat_id, bob_chat_id);
         assert_eq!(bob_msg.viewtype, Viewtype::Image);
 

src/login_param.rs

@@ -14,8 +14,22 @@ use crate::socks::Socks5Config;
 #[repr(u32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
-    /// Same as AcceptInvalidCertificates unless overridden by
-    /// `strict_tls` setting in provider database.
+    /// Same as AcceptInvalidCertificates if stored in the database
+    /// as `configured_{imap,smtp}_certificate_checks`.
+    ///
+    /// Previous Delta Chat versions stored this in `configured_*`
+    /// if Automatic configuration
+    /// was selected, configuration with strict TLS checks failed
+    /// and configuration without strict TLS checks succeeded.
+    ///
+    /// Currently Delta Chat stores only
+    /// `Strict` or `AcceptInvalidCertificates` variants
+    /// in `configured_*` settings.
+    ///
+    /// `Automatic` in `{imap,smtp}_certificate_checks`
+    /// means that provider database setting should be taken.
+    /// If there is no provider database setting for certificate checks,
+    /// `Automatic` is the same as `Strict`.
     Automatic = 0,
 
     Strict = 1,

src/message.rs

@@ -1663,9 +1663,17 @@ pub(crate) async fn update_msg_state(
     msg_id: MsgId,
     state: MessageState,
 ) -> Result<()> {
+    ensure!(state != MessageState::OutFailed, "use set_msg_failed()!");
+    let error_subst = match state >= MessageState::OutPending {
+        true => ", error=''",
+        false => "",
+    };
     context
         .sql
-        .execute("UPDATE msgs SET state=? WHERE id=?;", (state, msg_id))
+        .execute(
+            &format!("UPDATE msgs SET state=?1 {error_subst} WHERE id=?2 AND (?1!=?3 OR state<?3)"),
+            (state, msg_id, MessageState::OutDelivered),
+        )
         .await?;
     Ok(())
 }
@@ -1842,6 +1850,24 @@ pub(crate) async fn rfc724_mid_exists_and(
     Ok(res)
 }
 
+/// Given a list of Message-IDs, returns the latest message found in the database.
+///
+/// Only messages that are not in the trash chat are considered.
+pub(crate) async fn get_latest_by_rfc724_mids(
+    context: &Context,
+    mids: &[String],
+) -> Result<Option<Message>> {
+    for id in mids.iter().rev() {
+        if let Some(msg_id) = rfc724_mid_exists(context, id).await? {
+            let msg = Message::load_from_db(context, msg_id).await?;
+            if msg.chat_id != DC_CHAT_ID_TRASH {
+                return Ok(Some(msg));
+            }
+        }
+    }
+    Ok(None)
+}
+
 /// How a message is primarily displayed.
 #[derive(
     Debug,