Merge tag 'v1.124.0'

.github/workflows/ci.yml

@@ -180,15 +180,15 @@ jobs:
         include:
           # Currently used Rust version.
           - os: ubuntu-latest
-            python: 3.11
+            python: 3.12
           - os: macos-latest
-            python: 3.11
+            python: 3.12
 
           # PyPy tests
           - os: ubuntu-latest
-            python: pypy3.9
+            python: pypy3.10
           - os: macos-latest
-            python: pypy3.9
+            python: pypy3.10
 
           # Minimum Supported Python Version = 3.7
           # This is the minimum version for which manylinux Python wheels are
@@ -229,6 +229,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # Async Python bindings do not depend on Python version,
+          # but are tested on Python 3.11 until Python 3.12 support
+          # is added to `aiohttp` dependency:
+          # https://github.com/aio-libs/aiohttp/issues/7646
           - os: ubuntu-latest
             python: 3.11
           - os: macos-latest
@@ -236,9 +240,9 @@ jobs:
 
           # PyPy tests
           - os: ubuntu-latest
-            python: pypy3.9
+            python: pypy3.10
           - os: macos-latest
-            python: pypy3.9
+            python: pypy3.10
 
           # Minimum Supported Python Version = 3.8
           #

CHANGELOG.md

@@ -1,5 +1,38 @@
 # Changelog
 
+## [1.124.0] - 2023-10-04
+
+### API-Changes
+
+- [**breaking**] Return `DC_CONTACT_ID_SELF` from `dc_contact_get_verifier_id()` for directly verified contacts.
+- Deprecate `dc_contact_get_verifier_addr`.
+- python: use `dc_contact_get_verifier_id()`. `get_verifier()` returns a Contact rather than an address now.
+- Deprecate `get_next_media()`.
+- Ignore public key argument in `dc_preconfigure_keypair()`. Public key is extracted from the private key.
+
+### Fixes
+
+- Wrap base64-encoded parts to 76 characters.
+- Require valid email addresses in `dc_provider_new_from_email[_with_dns]`.
+- Do not trash messages with attachments and no text when `location.kml` is attached ([#4749](https://github.com/deltachat/deltachat-core-rust/issues/4749)).
+- Initialise `last_msg_id` to the highest known row id. This ensures bots migrated from older version to `dc_get_next_msgs()` API do not process all previous messages from scratch.
+- Do not put the status footer into reaction MIME parts.
+- Ignore special chats in `get_similar_chat_ids()`. This prevents trash chat from showing up in similar chat list ([#4756](https://github.com/deltachat/deltachat-core-rust/issues/4756)).
+- Cap percentage in connectivity layout to 100% ([#4765](https://github.com/deltachat/deltachat-core-rust/pull/4765)).
+- Add Let's Encrypt root certificate to `reqwest`. This should allow scanning `DCACCOUNT` QR-codes on older Android phones when the server has a Let's Encrypt certificate.
+- deltachat-rpc-client: Increase stdio buffer to 64 MiB to avoid Python bots crashing when trying to load large messages via a JSON-RPC call.
+- Add `protected-headers` directive to Content-Type of encrypted messages with attachments ([#2302](https://github.com/deltachat/deltachat-core-rust/issues/2302)). This makes Thunderbird show encrypted Subject for Delta Chat messages.
+- webxdc: Reset `document.update` on forwarding. This fixes the test `test_forward_webxdc_instance()`.
+
+### Features / Changes
+
+- Remove extra members from the local list in sake of group membership consistency ([#3782](https://github.com/deltachat/deltachat-core-rust/issues/3782)).
+- deltachat-rpc-client: Log exceptions when long-running tasks die.
+
+### Build
+
+- Build wheels for Python 3.12 and PyPy 3.10.
+
 ## [1.123.0] - 2023-09-22
 
 ### API-Changes
@@ -2834,3 +2867,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.121.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.120.0...v1.121.0
 [1.122.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.121.0...v1.122.0
 [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0
+[1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0

Cargo.lock

@@ -1085,7 +1085,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.123.0"
+version = "1.124.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1162,7 +1162,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.123.0"
+version = "1.124.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -1186,7 +1186,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "1.123.0"
+version = "1.124.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1201,7 +1201,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.123.0"
+version = "1.124.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1226,7 +1226,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.123.0"
+version = "1.124.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1594,7 +1594,7 @@ dependencies = [
 [[package]]
 name = "email"
 version = "0.0.21"
-source = "git+https://github.com/deltachat/rust-email?branch=master#25702df99254d059483b41417cd80696a258df8e"
+source = "git+https://github.com/deltachat/rust-email?branch=master#37778c89d5eb5a94b7983f3f37ff67769bde3cf9"
 dependencies = [
  "base64 0.11.0",
  "chrono",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.123.0"
+version = "1.124.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.67"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.123.0"
+version = "1.124.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -1488,6 +1488,7 @@ dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t ch
  * Typically used to implement the "next" and "previous" buttons
  * in a gallery or in a media player.
  *
+ * @deprecated Deprecated 2023-10-03, use dc_get_chat_media() and navigate the returned array instead.
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param msg_id The ID of the current message from which the next or previous message should be searched.

deltachat-ffi/src/lib.rs

@@ -1422,6 +1422,7 @@ pub unsafe extern "C" fn dc_get_chat_media(
 }
 
 #[no_mangle]
+#[allow(deprecated)]
 pub unsafe extern "C" fn dc_get_next_media(
     context: *mut dc_context_t,
     msg_id: u32,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.123.0"
+version = "1.124.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/src/api/mod.rs

@@ -1430,6 +1430,10 @@ impl CommandApi {
     ///
     /// one combined call for getting chat::get_next_media for both directions
     /// the manual chat::get_next_media in only one direction is not exposed by the jsonrpc yet
+    ///
+    /// Deprecated 2023-10-03, use `get_chat_media` method
+    /// and navigate the returned array instead.
+    #[allow(deprecated)]
     async fn get_neighboring_chat_media(
         &self,
         account_id: u32,

deltachat-jsonrpc/typescript/package.json

@@ -55,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.123.0"
+  "version": "1.124.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.123.0"
+version = "1.124.0"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+import logging
 import os
 from typing import Any, Dict, Optional
 
@@ -28,10 +29,16 @@ class Rpc:
         self.events_task: asyncio.Task
 
     async def start(self) -> None:
+        # Use buffer of 64 MiB.
+        # Default limit as of Python 3.11 is 2**16 bytes, this is too low for some JSON-RPC responses,
+        # such as loading large HTML message content.
+        limit = 2**26
+
         self.process = await asyncio.create_subprocess_exec(
             "deltachat-rpc-server",
             stdin=asyncio.subprocess.PIPE,
             stdout=asyncio.subprocess.PIPE,
+            limit=limit,
             **self._kwargs,
         )
         self.id = 0
@@ -57,16 +64,20 @@ class Rpc:
         await self.close()
 
     async def reader_loop(self) -> None:
-        while True:
+        try:
-            line = await self.process.stdout.readline()  # noqa
+            while True:
-            if not line:  # EOF
+                line = await self.process.stdout.readline()  # noqa
-                break
+                if not line:  # EOF
-            response = json.loads(line)
+                    break
-            if "id" in response:
+                response = json.loads(line)
-                fut = self.request_events.pop(response["id"])
+                if "id" in response:
-                fut.set_result(response)
+                    fut = self.request_events.pop(response["id"])
-            else:
+                    fut.set_result(response)
-                print(response)
+                else:
+                    print(response)
+        except Exception:
+            # Log an exception if the reader loop dies.
+            logging.exception("Exception in the reader loop")
 
     async def get_queue(self, account_id: int) -> asyncio.Queue:
         if account_id not in self.event_queues:
@@ -75,13 +86,17 @@ class Rpc:
 
     async def events_loop(self) -> None:
         """Requests new events and distributes them between queues."""
-        while True:
+        try:
-            if self.closing:
+            while True:
-                return
+                if self.closing:
-            event = await self.get_next_event()
+                    return
-            account_id = event["contextId"]
+                event = await self.get_next_event()
-            queue = await self.get_queue(account_id)
+                account_id = event["contextId"]
-            await queue.put(event["event"])
+                queue = await self.get_queue(account_id)
+                await queue.put(event["event"])
+        except Exception:
+            # Log an exception if the event loop dies.
+            logging.exception("Exception in the event loop")
 
     async def wait_for_event(self, account_id: int) -> Optional[dict]:
         """Waits for the next event from the given account and returns it."""

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.123.0"
+version = "1.124.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.123.0"
+  "version": "1.124.0"
 }

python/mypy.ini

@@ -24,3 +24,5 @@ ignore_missing_imports = True
 [mypy-imap_tools.*]
 ignore_missing_imports = True
 
+[mypy-distutils.*]
+ignore_missing_imports = True

release-date.in

@@ -1 +1 @@
-2023-09-22
+2023-10-04

scripts/run_all.sh

@@ -31,7 +31,7 @@ unset DCC_NEW_TMP_EMAIL
 
 # Try to build wheels for a range of interpreters, but don't fail if they are not available.
 # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,pypy37,pypy38,pypy39 --skip-missing-interpreters true
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true
 
 auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse"
 

src/chat.rs

@@ -932,9 +932,10 @@ impl ChatId {
             .sql
             .query_map(
                 "SELECT chat_id, count(*) AS n
-                 FROM chats_contacts where contact_id > 9
+                 FROM chats_contacts
+                 WHERE contact_id > ? AND chat_id > ?
                  GROUP BY chat_id",
-                (),
+                (ContactId::LAST_SPECIAL, DC_CHAT_ID_LAST_SPECIAL),
                 |row| {
                     let chat_id: ChatId = row.get(0)?;
                     let size: f64 = row.get(1)?;
@@ -2951,6 +2952,9 @@ pub enum Direction {
 }
 
 /// Searches next/previous message based on the given message and list of types.
+///
+/// Deprecated since 2023-10-03.
+#[deprecated(note = "use `get_chat_media` instead")]
 pub async fn get_next_media(
     context: &Context,
     curr_msg_id: MsgId,

src/contact.rs

@@ -109,7 +109,7 @@ impl ContactId {
 
     /// ID of the contact for device messages.
     pub const DEVICE: ContactId = ContactId::new(5);
-    const LAST_SPECIAL: ContactId = ContactId::new(9);
+    pub(crate) const LAST_SPECIAL: ContactId = ContactId::new(9);
 
     /// Address to go with [`ContactId::DEVICE`].
     ///

src/mimefactory.rs

@@ -678,6 +678,12 @@ impl<'a> MimeFactory<'a> {
                 })
         };
 
+        let get_content_type_directives_header = || {
+            (
+                "Content-Type-Deltachat-Directives".to_string(),
+                "protected-headers=\"v1\"".to_string(),
+            )
+        };
         let outer_message = if is_encrypted {
             headers.protected.push(from_header);
 
@@ -714,10 +720,7 @@ impl<'a> MimeFactory<'a> {
             if !existing_ct.ends_with(';') {
                 existing_ct += ";";
             }
-            let message = message.replace_header(Header::new(
+            let message = message.header(get_content_type_directives_header());
-                "Content-Type".to_string(),
-                format!("{existing_ct} protected-headers=\"v1\";"),
-            ));
 
             // Set the appropriate Content-Type for the outer message
             let outer_message = PartBuilder::new().header((
@@ -786,11 +789,12 @@ impl<'a> MimeFactory<'a> {
             {
                 message
             } else {
+                let message = message.header(get_content_type_directives_header());
                 let (payload, signature) = encrypt_helper.sign(context, message).await?;
                 PartBuilder::new()
                     .header((
-                        "Content-Type".to_string(),
+                        "Content-Type",
-                        "multipart/signed; protocol=\"application/pgp-signature\"".to_string(),
+                        "multipart/signed; protocol=\"application/pgp-signature\"",
                     ))
                     .child(payload)
                     .child(
@@ -1544,6 +1548,7 @@ fn maybe_encode_words(words: &str) -> String {
 #[cfg(test)]
 mod tests {
     use mailparse::{addrparse_header, MailHeaderMap};
+    use std::str;
 
     use super::*;
     use crate::chat::ChatId;
@@ -1552,10 +1557,11 @@ mod tests {
         ProtectionStatus,
     };
     use crate::chatlist::Chatlist;
+    use crate::constants;
     use crate::contact::{ContactAddress, Origin};
     use crate::mimeparser::MimeMessage;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::{get_chat_msg, TestContext};
+    use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
     #[test]
     fn test_render_email_address() {
         let display_name = "ä space";
@@ -2204,7 +2210,11 @@ mod tests {
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
 
         let part = payload.next().unwrap();
-        assert_eq!(part.match_indices("multipart/mixed").count(), 1);
+        assert_eq!(
+            part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+                .count(),
+            1
+        );
         assert_eq!(part.match_indices("Subject:").count(), 1);
         assert_eq!(part.match_indices("Autocrypt:").count(), 0);
         assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0);
@@ -2316,4 +2326,37 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_protected_headers_directive() -> Result<()> {
+        let mut tcm = TestContextManager::new();
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+        let chat = tcm
+            .send_recv_accept(&alice, &bob, "alice->bob")
+            .await
+            .chat_id;
+
+        // Now Bob can send an encrypted message to Alice.
+        let mut msg = Message::new(Viewtype::File);
+        // Long messages are truncated and MimeMessage::decoded_data is set for them. We need
+        // decoded_data to check presense of the necessary headers.
+        msg.set_text("a".repeat(constants::DC_DESIRED_TEXT_LEN + 1));
+        msg.set_file_from_bytes(&bob, "foo.bar", "content".as_bytes(), None)
+            .await?;
+        let sent = bob.send_msg(chat, &mut msg).await;
+        assert!(msg.get_showpadlock());
+
+        let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?;
+        let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n");
+        let part = payload.next().unwrap();
+        assert_eq!(
+            part.match_indices("multipart/mixed; protected-headers=\"v1\"")
+                .count(),
+            1
+        );
+        assert_eq!(part.match_indices("Subject:").count(), 1);
+
+        Ok(())
+    }
 }

src/net/http.rs

@@ -4,12 +4,20 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Result};
 use mime::Mime;
+use once_cell::sync::Lazy;
 
 use crate::context::Context;
 use crate::socks::Socks5Config;
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
 
+static LETSENCRYPT_ROOT: Lazy<reqwest::tls::Certificate> = Lazy::new(|| {
+    reqwest::tls::Certificate::from_der(include_bytes!(
+        "../../assets/root-certificates/letsencrypt/isrgrootx1.der"
+    ))
+    .unwrap()
+});
+
 /// HTTP(S) GET response.
 #[derive(Debug)]
 pub struct Response {
@@ -79,7 +87,10 @@ async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Respons
 }
 
 pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
-    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
+    let builder = reqwest::ClientBuilder::new()
+        .timeout(HTTP_TIMEOUT)
+        .add_root_certificate(LETSENCRYPT_ROOT.clone());
+
     let builder = if let Some(socks5_config) = socks5_config {
         let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
         builder.proxy(proxy)

src/scheduler/connectivity.rs

@@ -1,4 +1,5 @@
 use core::fmt;
+use std::cmp::min;
 use std::{iter::once, ops::Deref, sync::Arc};
 
 use anyhow::{anyhow, Result};
@@ -457,7 +458,8 @@ impl Context {
                                 } else {
                                     "green"
                                 };
-                                ret += &format!("<div class=\"bar\"><div class=\"progress {color}\" style=\"width: {percent}%\">{percent}%</div></div>");
+                                let div_width_percent = min(100, percent);
+                                ret += &format!("<div class=\"bar\"><div class=\"progress {color}\" style=\"width: {div_width_percent}%\">{percent}%</div></div>");
 
                                 ret += "</li>";
                             }