diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6bb5f6e0..e331ddf94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,15 +180,15 @@ jobs: include: # Currently used Rust version. - os: ubuntu-latest - python: 3.11 + python: 3.12 - os: macos-latest - python: 3.11 + python: 3.12 # PyPy tests - os: ubuntu-latest - python: pypy3.9 + python: pypy3.10 - os: macos-latest - python: pypy3.9 + python: pypy3.10 # Minimum Supported Python Version = 3.7 # This is the minimum version for which manylinux Python wheels are @@ -229,6 +229,10 @@ jobs: fail-fast: false matrix: include: + # Async Python bindings do not depend on Python version, + # but are tested on Python 3.11 until Python 3.12 support + # is added to `aiohttp` dependency: + # https://github.com/aio-libs/aiohttp/issues/7646 - os: ubuntu-latest python: 3.11 - os: macos-latest @@ -236,9 +240,9 @@ jobs: # PyPy tests - os: ubuntu-latest - python: pypy3.9 + python: pypy3.10 - os: macos-latest - python: pypy3.9 + python: pypy3.10 # Minimum Supported Python Version = 3.8 # diff --git a/CHANGELOG.md b/CHANGELOG.md index f6ea5ea5b..3ab0cc53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## [1.124.0] - 2023-10-04 + +### API-Changes + +- [**breaking**] Return `DC_CONTACT_ID_SELF` from `dc_contact_get_verifier_id()` for directly verified contacts. +- Deprecate `dc_contact_get_verifier_addr`. +- python: use `dc_contact_get_verifier_id()`. `get_verifier()` returns a Contact rather than an address now. +- Deprecate `get_next_media()`. +- Ignore public key argument in `dc_preconfigure_keypair()`. Public key is extracted from the private key. + +### Fixes + +- Wrap base64-encoded parts to 76 characters. +- Require valid email addresses in `dc_provider_new_from_email[_with_dns]`. +- Do not trash messages with attachments and no text when `location.kml` is attached ([#4749](https://github.com/deltachat/deltachat-core-rust/issues/4749)). +- Initialise `last_msg_id` to the highest known row id. This ensures bots migrated from older version to `dc_get_next_msgs()` API do not process all previous messages from scratch. +- Do not put the status footer into reaction MIME parts. +- Ignore special chats in `get_similar_chat_ids()`. This prevents trash chat from showing up in similar chat list ([#4756](https://github.com/deltachat/deltachat-core-rust/issues/4756)). +- Cap percentage in connectivity layout to 100% ([#4765](https://github.com/deltachat/deltachat-core-rust/pull/4765)). +- Add Let's Encrypt root certificate to `reqwest`. This should allow scanning `DCACCOUNT` QR-codes on older Android phones when the server has a Let's Encrypt certificate. +- deltachat-rpc-client: Increase stdio buffer to 64 MiB to avoid Python bots crashing when trying to load large messages via a JSON-RPC call. +- Add `protected-headers` directive to Content-Type of encrypted messages with attachments ([#2302](https://github.com/deltachat/deltachat-core-rust/issues/2302)). This makes Thunderbird show encrypted Subject for Delta Chat messages. +- webxdc: Reset `document.update` on forwarding. This fixes the test `test_forward_webxdc_instance()`. + +### Features / Changes + +- Remove extra members from the local list in sake of group membership consistency ([#3782](https://github.com/deltachat/deltachat-core-rust/issues/3782)). +- deltachat-rpc-client: Log exceptions when long-running tasks die. + +### Build + +- Build wheels for Python 3.12 and PyPy 3.10. + ## [1.123.0] - 2023-09-22 ### API-Changes @@ -2834,3 +2867,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed [1.121.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.120.0...v1.121.0 [1.122.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.121.0...v1.122.0 [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0 +[1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0 diff --git a/Cargo.lock b/Cargo.lock index 0ec49736a..0bbf4cc60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.123.0" +version = "1.124.0" dependencies = [ "ansi_term", "anyhow", @@ -1162,7 +1162,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.123.0" +version = "1.124.0" dependencies = [ "anyhow", "async-channel", @@ -1186,7 +1186,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.123.0" +version = "1.124.0" dependencies = [ "ansi_term", "anyhow", @@ -1201,7 +1201,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.123.0" +version = "1.124.0" dependencies = [ "anyhow", "deltachat", @@ -1226,7 +1226,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.123.0" +version = "1.124.0" dependencies = [ "anyhow", "deltachat", @@ -1594,7 +1594,7 @@ dependencies = [ [[package]] name = "email" version = "0.0.21" -source = "git+https://github.com/deltachat/rust-email?branch=master#25702df99254d059483b41417cd80696a258df8e" +source = "git+https://github.com/deltachat/rust-email?branch=master#37778c89d5eb5a94b7983f3f37ff67769bde3cf9" dependencies = [ "base64 0.11.0", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 278bdb1cf..9b1664ca6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.123.0" +version = "1.124.0" edition = "2021" license = "MPL-2.0" rust-version = "1.67" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index b544dbb57..03628e0ce 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.123.0" +version = "1.124.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 7e83e181f..4988f1e97 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1488,6 +1488,7 @@ dc_array_t* dc_get_chat_media (dc_context_t* context, uint32_t ch * Typically used to implement the "next" and "previous" buttons * in a gallery or in a media player. * + * @deprecated Deprecated 2023-10-03, use dc_get_chat_media() and navigate the returned array instead. * @memberof dc_context_t * @param context The context object as returned from dc_context_new(). * @param msg_id The ID of the current message from which the next or previous message should be searched. diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 689300d23..025c3f7ce 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -1422,6 +1422,7 @@ pub unsafe extern "C" fn dc_get_chat_media( } #[no_mangle] +#[allow(deprecated)] pub unsafe extern "C" fn dc_get_next_media( context: *mut dc_context_t, msg_id: u32, diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index bab37cd56..eb9335f01 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.123.0" +version = "1.124.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index 8f6293bfb..6b9e8d5ff 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -1430,6 +1430,10 @@ impl CommandApi { /// /// one combined call for getting chat::get_next_media for both directions /// the manual chat::get_next_media in only one direction is not exposed by the jsonrpc yet + /// + /// Deprecated 2023-10-03, use `get_chat_media` method + /// and navigate the returned array instead. + #[allow(deprecated)] async fn get_neighboring_chat_media( &self, account_id: u32, diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 7f0307837..65848a35f 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -55,5 +55,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.123.0" + "version": "1.124.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index ab623efbb..f21984d3e 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.123.0" +version = "1.124.0" license = "MPL-2.0" edition = "2021" diff --git a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py index df65ffea1..a3df4723e 100644 --- a/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py +++ b/deltachat-rpc-client/src/deltachat_rpc_client/rpc.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import os from typing import Any, Dict, Optional @@ -28,10 +29,16 @@ class Rpc: self.events_task: asyncio.Task async def start(self) -> None: + # Use buffer of 64 MiB. + # Default limit as of Python 3.11 is 2**16 bytes, this is too low for some JSON-RPC responses, + # such as loading large HTML message content. + limit = 2**26 + self.process = await asyncio.create_subprocess_exec( "deltachat-rpc-server", stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, + limit=limit, **self._kwargs, ) self.id = 0 @@ -57,16 +64,20 @@ class Rpc: await self.close() async def reader_loop(self) -> None: - while True: - line = await self.process.stdout.readline() # noqa - if not line: # EOF - break - response = json.loads(line) - if "id" in response: - fut = self.request_events.pop(response["id"]) - fut.set_result(response) - else: - print(response) + try: + while True: + line = await self.process.stdout.readline() # noqa + if not line: # EOF + break + response = json.loads(line) + if "id" in response: + fut = self.request_events.pop(response["id"]) + fut.set_result(response) + else: + print(response) + except Exception: + # Log an exception if the reader loop dies. + logging.exception("Exception in the reader loop") async def get_queue(self, account_id: int) -> asyncio.Queue: if account_id not in self.event_queues: @@ -75,13 +86,17 @@ class Rpc: async def events_loop(self) -> None: """Requests new events and distributes them between queues.""" - while True: - if self.closing: - return - event = await self.get_next_event() - account_id = event["contextId"] - queue = await self.get_queue(account_id) - await queue.put(event["event"]) + try: + while True: + if self.closing: + return + event = await self.get_next_event() + account_id = event["contextId"] + queue = await self.get_queue(account_id) + await queue.put(event["event"]) + except Exception: + # Log an exception if the event loop dies. + logging.exception("Exception in the event loop") async def wait_for_event(self, account_id: int) -> Optional[dict]: """Waits for the next event from the given account and returns it.""" diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 994d5e963..a3494b694 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.123.0" +version = "1.124.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 31743b738..edec01fc9 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.123.0" + "version": "1.124.0" } diff --git a/python/mypy.ini b/python/mypy.ini index 6b7560ab9..3eb9ae8c8 100644 --- a/python/mypy.ini +++ b/python/mypy.ini @@ -24,3 +24,5 @@ ignore_missing_imports = True [mypy-imap_tools.*] ignore_missing_imports = True +[mypy-distutils.*] +ignore_missing_imports = True diff --git a/release-date.in b/release-date.in index d6d1f7fd5..0539de2f7 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2023-09-22 \ No newline at end of file +2023-10-04 \ No newline at end of file diff --git a/scripts/run_all.sh b/scripts/run_all.sh index 87ce41ea3..42edb094f 100755 --- a/scripts/run_all.sh +++ b/scripts/run_all.sh @@ -31,7 +31,7 @@ unset DCC_NEW_TMP_EMAIL # Try to build wheels for a range of interpreters, but don't fail if they are not available. # E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10 -tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,pypy37,pypy38,pypy39 --skip-missing-interpreters true +tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,py311,py312,pypy37,pypy38,pypy39,pypy310 --skip-missing-interpreters true auditwheel repair "$TOXWORKDIR"/wheelhouse/deltachat* -w "$TOXWORKDIR/wheelhouse" diff --git a/src/chat.rs b/src/chat.rs index c260c7786..7687e554e 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -932,9 +932,10 @@ impl ChatId { .sql .query_map( "SELECT chat_id, count(*) AS n - FROM chats_contacts where contact_id > 9 + FROM chats_contacts + WHERE contact_id > ? AND chat_id > ? GROUP BY chat_id", - (), + (ContactId::LAST_SPECIAL, DC_CHAT_ID_LAST_SPECIAL), |row| { let chat_id: ChatId = row.get(0)?; let size: f64 = row.get(1)?; @@ -2951,6 +2952,9 @@ pub enum Direction { } /// Searches next/previous message based on the given message and list of types. +/// +/// Deprecated since 2023-10-03. +#[deprecated(note = "use `get_chat_media` instead")] pub async fn get_next_media( context: &Context, curr_msg_id: MsgId, diff --git a/src/contact.rs b/src/contact.rs index 6170fd0e3..161fb571e 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -109,7 +109,7 @@ impl ContactId { /// ID of the contact for device messages. pub const DEVICE: ContactId = ContactId::new(5); - const LAST_SPECIAL: ContactId = ContactId::new(9); + pub(crate) const LAST_SPECIAL: ContactId = ContactId::new(9); /// Address to go with [`ContactId::DEVICE`]. /// diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 09f6f59f0..46d4b1ed0 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -678,6 +678,12 @@ impl<'a> MimeFactory<'a> { }) }; + let get_content_type_directives_header = || { + ( + "Content-Type-Deltachat-Directives".to_string(), + "protected-headers=\"v1\"".to_string(), + ) + }; let outer_message = if is_encrypted { headers.protected.push(from_header); @@ -714,10 +720,7 @@ impl<'a> MimeFactory<'a> { if !existing_ct.ends_with(';') { existing_ct += ";"; } - let message = message.replace_header(Header::new( - "Content-Type".to_string(), - format!("{existing_ct} protected-headers=\"v1\";"), - )); + let message = message.header(get_content_type_directives_header()); // Set the appropriate Content-Type for the outer message let outer_message = PartBuilder::new().header(( @@ -786,11 +789,12 @@ impl<'a> MimeFactory<'a> { { message } else { + let message = message.header(get_content_type_directives_header()); let (payload, signature) = encrypt_helper.sign(context, message).await?; PartBuilder::new() .header(( - "Content-Type".to_string(), - "multipart/signed; protocol=\"application/pgp-signature\"".to_string(), + "Content-Type", + "multipart/signed; protocol=\"application/pgp-signature\"", )) .child(payload) .child( @@ -1544,6 +1548,7 @@ fn maybe_encode_words(words: &str) -> String { #[cfg(test)] mod tests { use mailparse::{addrparse_header, MailHeaderMap}; + use std::str; use super::*; use crate::chat::ChatId; @@ -1552,10 +1557,11 @@ mod tests { ProtectionStatus, }; use crate::chatlist::Chatlist; + use crate::constants; use crate::contact::{ContactAddress, Origin}; use crate::mimeparser::MimeMessage; use crate::receive_imf::receive_imf; - use crate::test_utils::{get_chat_msg, TestContext}; + use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; #[test] fn test_render_email_address() { let display_name = "รค space"; @@ -2204,7 +2210,11 @@ mod tests { assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); let part = payload.next().unwrap(); - assert_eq!(part.match_indices("multipart/mixed").count(), 1); + assert_eq!( + part.match_indices("multipart/mixed; protected-headers=\"v1\"") + .count(), + 1 + ); assert_eq!(part.match_indices("Subject:").count(), 1); assert_eq!(part.match_indices("Autocrypt:").count(), 0); assert_eq!(part.match_indices("Chat-User-Avatar:").count(), 0); @@ -2316,4 +2326,37 @@ mod tests { Ok(()) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_protected_headers_directive() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; + let chat = tcm + .send_recv_accept(&alice, &bob, "alice->bob") + .await + .chat_id; + + // Now Bob can send an encrypted message to Alice. + let mut msg = Message::new(Viewtype::File); + // Long messages are truncated and MimeMessage::decoded_data is set for them. We need + // decoded_data to check presense of the necessary headers. + msg.set_text("a".repeat(constants::DC_DESIRED_TEXT_LEN + 1)); + msg.set_file_from_bytes(&bob, "foo.bar", "content".as_bytes(), None) + .await?; + let sent = bob.send_msg(chat, &mut msg).await; + assert!(msg.get_showpadlock()); + + let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?; + let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n"); + let part = payload.next().unwrap(); + assert_eq!( + part.match_indices("multipart/mixed; protected-headers=\"v1\"") + .count(), + 1 + ); + assert_eq!(part.match_indices("Subject:").count(), 1); + + Ok(()) + } } diff --git a/src/net/http.rs b/src/net/http.rs index cbee57959..6f31e3348 100644 --- a/src/net/http.rs +++ b/src/net/http.rs @@ -4,12 +4,20 @@ use std::time::Duration; use anyhow::{anyhow, Result}; use mime::Mime; +use once_cell::sync::Lazy; use crate::context::Context; use crate::socks::Socks5Config; const HTTP_TIMEOUT: Duration = Duration::from_secs(30); +static LETSENCRYPT_ROOT: Lazy = Lazy::new(|| { + reqwest::tls::Certificate::from_der(include_bytes!( + "../../assets/root-certificates/letsencrypt/isrgrootx1.der" + )) + .unwrap() +}); + /// HTTP(S) GET response. #[derive(Debug)] pub struct Response { @@ -79,7 +87,10 @@ async fn read_url_inner(context: &Context, url: &str) -> Result) -> Result { - let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT); + let builder = reqwest::ClientBuilder::new() + .timeout(HTTP_TIMEOUT) + .add_root_certificate(LETSENCRYPT_ROOT.clone()); + let builder = if let Some(socks5_config) = socks5_config { let proxy = reqwest::Proxy::all(socks5_config.to_url())?; builder.proxy(proxy) diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index 472b10001..4d8a66b91 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -1,4 +1,5 @@ use core::fmt; +use std::cmp::min; use std::{iter::once, ops::Deref, sync::Arc}; use anyhow::{anyhow, Result}; @@ -457,7 +458,8 @@ impl Context { } else { "green" }; - ret += &format!("
{percent}%
"); + let div_width_percent = min(100, percent); + ret += &format!("
{percent}%
"); ret += ""; }