diff --git a/CHANGELOG.md b/CHANGELOG.md index 25740972d..8e54a6826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## [1.120.0] - 2023-08-28 + +### API-Changes + +- jsonrpc: Add `resend_messages`. + +### Fixes + +- Update async-imap to 0.9.1 to fix memory leak. +- Delete messages from SMTP queue only on user demand ([#4579](https://github.com/deltachat/deltachat-core-rust/pull/4579)). +- Do not send images without transparency as stickers ([#4611](https://github.com/deltachat/deltachat-core-rust/pull/4611)). +- `prepare_msg_blob()`: do not use the image if it has Exif metadata but the image cannot be recoded. + +### Refactor + +- Hide accounts.rs constants from public API. +- Hide pgp module from public API. + +### Build system + +- Update to Zig 0.11.0. +- Update to Rust 1.72.0. + +### CI + +- Run on push to stable branch. + +### Miscellaneous Tasks + +- python: Fix lint errors. +- python: Fix `ruff` 0.0.286 warnings. +- Fix beta clippy warnings. + ## [1.119.1] - 2023-08-06 Bugfix release attempting to fix the [iOS build error](https://github.com/deltachat/deltachat-core-rust/issues/4610). @@ -2730,3 +2763,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed [1.118.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.117.0...v1.118.0 [1.119.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.118.0...v1.119.0 [1.119.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.119.0...v1.119.1 +[1.120.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.119.1...v1.120.0 diff --git a/Cargo.lock b/Cargo.lock index aec4f1464..07fd501c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,13 +210,13 @@ dependencies = [ [[package]] name = "async-imap" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da93622739d458dd9a6abc1abf0e38e81965a5824a3b37f9500437c82a8bb572" +checksum = "b538b767cbf9c162a6c5795d4b932bd2c20ba10b5a91a94d2b2b6886c1dce6a8" dependencies = [ "async-channel", "base64 0.21.2", - "byte-pool", + "bytes", "chrono", "futures", "imap-proto", @@ -533,16 +533,6 @@ version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" -[[package]] -name = "byte-pool" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f1b21189f50b5625efa6227cf45e9d4cfdc2e73582df2b879e9689e78a7158" -dependencies = [ - "crossbeam-queue", - "stable_deref_trait", -] - [[package]] name = "bytemuck" version = "1.13.1" @@ -916,16 +906,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -1105,7 +1085,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.119.1" +version = "1.120.0" dependencies = [ "ansi_term", "anyhow", @@ -1182,7 +1162,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.119.1" +version = "1.120.0" dependencies = [ "anyhow", "async-channel", @@ -1206,7 +1186,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.119.1" +version = "1.120.0" dependencies = [ "ansi_term", "anyhow", @@ -1221,7 +1201,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.119.1" +version = "1.120.0" dependencies = [ "anyhow", "deltachat", @@ -1246,7 +1226,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.119.1" +version = "1.120.0" dependencies = [ "anyhow", "deltachat", @@ -4549,12 +4529,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "static_assertions" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5c2559f31..6b788c23f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.119.1" +version = "1.120.0" edition = "2021" license = "MPL-2.0" rust-version = "1.67" @@ -36,7 +36,7 @@ ratelimit = { path = "./deltachat-ratelimit" } anyhow = "1" async-channel = "1.8.0" -async-imap = { version = "0.9.0", default-features = false, features = ["runtime-tokio"] } +async-imap = { version = "0.9.1", default-features = false, features = ["runtime-tokio"] } async-native-tls = { version = "0.5", default-features = false, features = ["runtime-tokio"] } async-smtp = { version = "0.9", default-features = false, features = ["runtime-tokio"] } async_zip = { version = "0.0.12", default-features = false, features = ["deflate", "fs"] } diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 232199630..546fec81d 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.119.1" +version = "1.120.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index f60075101..f5812da18 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.119.1" +version = "1.120.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index c020bb4fa..0c0e4b6bd 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -55,5 +55,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.119.1" + "version": "1.120.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 4c2790391..32402d1d0 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.119.1" +version = "1.120.0" license = "MPL-2.0" edition = "2021" diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 98d859f8d..cf509dd2c 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.119.1" +version = "1.120.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 1c1c618fa..46f77993d 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.119.1" + "version": "1.120.0" } diff --git a/release-date.in b/release-date.in index a6535abb7..13b929383 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2023-08-06 \ No newline at end of file +2023-08-28 \ No newline at end of file diff --git a/src/blob.rs b/src/blob.rs index 579a2ad4e..40072cb53 100644 --- a/src/blob.rs +++ b/src/blob.rs @@ -9,7 +9,7 @@ use std::path::{Path, PathBuf}; use anyhow::{format_err, Context as _, Result}; use futures::StreamExt; -use image::{DynamicImage, ImageFormat, ImageOutputFormat}; +use image::{DynamicImage, GenericImageView, ImageFormat, ImageOutputFormat}; use num_traits::FromPrimitive; use tokio::io::AsyncWriteExt; use tokio::{fs, io}; @@ -323,18 +323,35 @@ impl<'a> BlobObject<'a> { MediaQuality::Worse => constants::WORSE_AVATAR_SIZE, }; + let maybe_sticker = &mut false; let strict_limits = true; // max_bytes is 20_000 bytes: Outlook servers don't allow headers larger than 32k. // 32 / 4 * 3 = 24k if you account for base64 encoding. To be safe, we reduced this to 20k. - if let Some(new_name) = - self.recode_to_size(context, blob_abs, img_wh, 20_000, strict_limits)? - { + if let Some(new_name) = self.recode_to_size( + context, + blob_abs, + maybe_sticker, + img_wh, + 20_000, + strict_limits, + )? { self.name = new_name; } Ok(()) } - pub async fn recode_to_image_size(&mut self, context: &Context) -> Result<()> { + /// Recodes an image pointed by a [BlobObject] so that it fits into limits on the image width, + /// height and file size specified by the config. + /// + /// On some platforms images are passed to the core as [`crate::message::Viewtype::Sticker`] in + /// which case `maybe_sticker` flag should be set. We recheck if an image is a true sticker + /// assuming that it must have at least one fully transparent corner, otherwise this flag is + /// reset. + pub async fn recode_to_image_size( + &mut self, + context: &Context, + maybe_sticker: &mut bool, + ) -> Result<()> { let blob_abs = self.to_abs_path(); let (img_wh, max_bytes) = match MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await?) @@ -347,9 +364,14 @@ impl<'a> BlobObject<'a> { MediaQuality::Worse => (constants::WORSE_IMAGE_SIZE, constants::WORSE_IMAGE_BYTES), }; let strict_limits = false; - if let Some(new_name) = - self.recode_to_size(context, blob_abs, img_wh, max_bytes, strict_limits)? - { + if let Some(new_name) = self.recode_to_size( + context, + blob_abs, + maybe_sticker, + img_wh, + max_bytes, + strict_limits, + )? { self.name = new_name; } Ok(()) @@ -358,20 +380,37 @@ impl<'a> BlobObject<'a> { /// If `!strict_limits`, then if `max_bytes` is exceeded, reduce the image to `img_wh` and just /// proceed with the result. fn recode_to_size( - &self, + &mut self, context: &Context, mut blob_abs: PathBuf, + maybe_sticker: &mut bool, mut img_wh: u32, max_bytes: usize, strict_limits: bool, ) -> Result> { - tokio::task::block_in_place(move || { - let mut img = image::open(&blob_abs).context("image decode failure")?; + let mut no_exif = false; + let no_exif_ref = &mut no_exif; + let res = tokio::task::block_in_place(move || { let (nr_bytes, exif) = self.metadata()?; + *no_exif_ref = exif.is_none(); + let mut img = image::open(&blob_abs).context("image decode failure")?; let orientation = exif.as_ref().map(|exif| exif_orientation(exif, context)); let mut encoded = Vec::new(); let mut changed_name = None; + if *maybe_sticker { + let x_max = img.width().saturating_sub(1); + let y_max = img.height().saturating_sub(1); + *maybe_sticker = img.in_bounds(x_max, y_max) + && (img.get_pixel(0, 0).0[3] == 0 + || img.get_pixel(x_max, 0).0[3] == 0 + || img.get_pixel(0, y_max).0[3] == 0 + || img.get_pixel(x_max, y_max).0[3] == 0); + } + if *maybe_sticker && exif.is_none() { + return Ok(None); + } + img = match orientation { Some(90) => img.rotate90(), Some(180) => img.rotate180(), @@ -469,7 +508,21 @@ impl<'a> BlobObject<'a> { } Ok(changed_name) - }) + }); + match res { + Ok(_) => res, + Err(err) => { + if !strict_limits && no_exif { + warn!( + context, + "Cannot recode image, using original data: {err:#}.", + ); + Ok(None) + } else { + Err(err) + } + } + } } /// Returns image file size and Exif. @@ -860,10 +913,18 @@ mod tests { file.metadata().await.unwrap().len() } - let blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap(); + let mut blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap(); + let maybe_sticker = &mut false; let strict_limits = true; - blob.recode_to_size(&t, blob.to_abs_path(), 1000, 3000, strict_limits) - .unwrap(); + blob.recode_to_size( + &t, + blob.to_abs_path(), + maybe_sticker, + 1000, + 3000, + strict_limits, + ) + .unwrap(); assert!(file_size(&avatar_blob).await <= 3000); assert!(file_size(&avatar_blob).await > 2000); tokio::task::block_in_place(move || { @@ -923,6 +984,7 @@ mod tests { async fn test_recode_image_1() { let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg"); send_image_check_mediaquality( + Viewtype::Image, Some("0"), bytes, "jpg", @@ -936,6 +998,7 @@ mod tests { .await .unwrap(); send_image_check_mediaquality( + Viewtype::Image, Some("1"), bytes, "jpg", @@ -955,6 +1018,7 @@ mod tests { // The "-rotated" files are rotated by 270 degrees using the Exif metadata let bytes = include_bytes!("../test-data/image/rectangle2000x1800-rotated.jpg"); let img_rotated = send_image_check_mediaquality( + Viewtype::Image, Some("0"), bytes, "jpg", @@ -974,6 +1038,7 @@ mod tests { let bytes = buf.into_inner(); let img_rotated = send_image_check_mediaquality( + Viewtype::Image, Some("1"), &bytes, "jpg", @@ -994,6 +1059,7 @@ mod tests { let bytes = include_bytes!("../test-data/image/screenshot.png"); send_image_check_mediaquality( + Viewtype::Image, Some("0"), bytes, "png", @@ -1008,6 +1074,7 @@ mod tests { .unwrap(); send_image_check_mediaquality( + Viewtype::Image, Some("1"), bytes, "png", @@ -1020,12 +1087,29 @@ mod tests { ) .await .unwrap(); + + // This will be sent as Image, see [`BlobObject::maybe_sticker`] for explanation. + send_image_check_mediaquality( + Viewtype::Sticker, + Some("0"), + bytes, + "png", + false, // no Exif + 1920, + 1080, + 0, + 1920, + 1080, + ) + .await + .unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_recode_image_huge_jpg() { let bytes = include_bytes!("../test-data/image/screenshot.jpg"); send_image_check_mediaquality( + Viewtype::Image, Some("0"), bytes, "jpg", @@ -1059,6 +1143,7 @@ mod tests { #[allow(clippy::too_many_arguments)] async fn send_image_check_mediaquality( + viewtype: Viewtype, media_quality_config: Option<&str>, bytes: &[u8], extension: &str, @@ -1090,7 +1175,7 @@ mod tests { assert!(exif.is_none()); } - let mut msg = Message::new(Viewtype::Image); + let mut msg = Message::new(viewtype); msg.set_file(file.to_str().unwrap(), None); let chat = alice.create_chat(&bob).await; let sent = alice.send_msg(chat.id, &mut msg).await; @@ -1104,6 +1189,7 @@ mod tests { ); let bob_msg = bob.recv_msg(&sent).await; + assert_eq!(bob_msg.get_viewtype(), Viewtype::Image); assert_eq!(bob_msg.get_width() as u32, compressed_width); assert_eq!(bob_msg.get_height() as u32, compressed_height); let file = bob_msg.get_file(&bob).unwrap(); diff --git a/src/chat.rs b/src/chat.rs index 9b4af936c..ed2c03cfe 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2082,12 +2082,12 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> { .await? .with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?; - if msg.viewtype == Viewtype::Image { - if let Err(err) = blob.recode_to_image_size(context).await { - warn!( - context, - "Cannot recode image, using original data: {err:#}." - ); + let mut maybe_sticker = msg.viewtype == Viewtype::Sticker; + if msg.viewtype == Viewtype::Image || maybe_sticker { + blob.recode_to_image_size(context, &mut maybe_sticker) + .await?; + if !maybe_sticker { + msg.viewtype = Viewtype::Image; } } msg.param.set(Param::File, blob.as_name()); @@ -5494,7 +5494,13 @@ mod tests { Ok(()) } - async fn test_sticker(filename: &str, bytes: &[u8], w: i32, h: i32) -> Result<()> { + async fn test_sticker( + filename: &str, + bytes: &[u8], + res_viewtype: Viewtype, + w: i32, + h: i32, + ) -> Result<()> { let alice = TestContext::new_alice().await; let bob = TestContext::new_bob().await; let alice_chat = alice.create_chat(&bob).await; @@ -5508,12 +5514,19 @@ mod tests { let sent_msg = alice.send_msg(alice_chat.id, &mut msg).await; let mime = sent_msg.payload(); - assert_eq!(mime.match_indices("Chat-Content: sticker").count(), 1); + if res_viewtype == Viewtype::Sticker { + assert_eq!(mime.match_indices("Chat-Content: sticker").count(), 1); + } let msg = bob.recv_msg(&sent_msg).await; assert_eq!(msg.chat_id, bob_chat.id); - assert_eq!(msg.get_viewtype(), Viewtype::Sticker); - assert_eq!(msg.get_filename().unwrap(), filename); + assert_eq!(msg.get_viewtype(), res_viewtype); + let msg_filename = msg.get_filename().unwrap(); + match res_viewtype { + Viewtype::Sticker => assert_eq!(msg_filename, filename), + Viewtype::Image => assert!(msg_filename.starts_with("image_")), + _ => panic!("Not implemented"), + } assert_eq!(msg.get_width(), w); assert_eq!(msg.get_height(), h); assert!(msg.get_filebytes(&bob).await?.unwrap() > 250); @@ -5525,9 +5538,10 @@ mod tests { async fn test_sticker_png() -> Result<()> { test_sticker( "sticker.png", - include_bytes!("../test-data/image/avatar64x64.png"), - 64, - 64, + include_bytes!("../test-data/image/logo.png"), + Viewtype::Sticker, + 135, + 135, ) .await } @@ -5537,6 +5551,7 @@ mod tests { test_sticker( "sticker.jpg", include_bytes!("../test-data/image/avatar1000x1000.jpg"), + Viewtype::Image, 1000, 1000, ) @@ -5547,9 +5562,10 @@ mod tests { async fn test_sticker_gif() -> Result<()> { test_sticker( "sticker.gif", - include_bytes!("../test-data/image/image100x50.gif"), - 100, - 50, + include_bytes!("../test-data/image/logo.gif"), + Viewtype::Sticker, + 135, + 135, ) .await } @@ -5563,8 +5579,8 @@ mod tests { let bob_chat = bob.create_chat(&alice).await; // create sticker - let file_name = "sticker.jpg"; - let bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg"); + let file_name = "sticker.png"; + let bytes = include_bytes!("../test-data/image/logo.png"); let file = alice.get_blobdir().join(file_name); tokio::fs::write(&file, bytes).await?; let mut msg = Message::new(Viewtype::Sticker); @@ -6101,7 +6117,7 @@ mod tests { chat_id1, Viewtype::Sticker, "b.png", - include_bytes!("../test-data/image/avatar64x64.png"), + include_bytes!("../test-data/image/logo.png"), ) .await?; let second_image_msg_id = send_media( diff --git a/src/job.rs b/src/job.rs index f739f4f63..d1ae367ea 100644 --- a/src/job.rs +++ b/src/job.rs @@ -155,8 +155,8 @@ impl<'a> Connection<'a> { pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_>, mut job: Job) { info!(context, "Job {} started...", &job); - let try_res = match perform_job_action(context, &mut job, &mut connection, 0).await { - Status::RetryNow => perform_job_action(context, &mut job, &mut connection, 1).await, + let try_res = match perform_job_action(context, &job, &mut connection, 0).await { + Status::RetryNow => perform_job_action(context, &job, &mut connection, 1).await, x => x, }; @@ -205,7 +205,7 @@ pub(crate) async fn perform_job(context: &Context, mut connection: Connection<'_ async fn perform_job_action( context: &Context, - job: &mut Job, + job: &Job, connection: &mut Connection<'_>, tries: u32, ) -> Status { diff --git a/src/quota.rs b/src/quota.rs index cc3f428fd..f2a4b921d 100644 --- a/src/quota.rs +++ b/src/quota.rs @@ -71,7 +71,7 @@ async fn get_unique_quota_roots_and_usage( // messages could be received and so the usage could have been changed *unique_quota_roots .entry(quota_root_name.clone()) - .or_insert_with(Vec::new) = quota.resources; + .or_default() = quota.resources; } } } diff --git a/src/webxdc.rs b/src/webxdc.rs index 6a89f097d..a47f3d8e0 100644 --- a/src/webxdc.rs +++ b/src/webxdc.rs @@ -707,7 +707,7 @@ fn parse_webxdc_manifest(bytes: &[u8]) -> Result { Ok(manifest) } -async fn get_blob(archive: &mut async_zip::read::fs::ZipFileReader, name: &str) -> Result> { +async fn get_blob(archive: &async_zip::read::fs::ZipFileReader, name: &str) -> Result> { let (i, _) = find_zip_entry(archive.file(), name) .ok_or_else(|| anyhow!("no entry found for {}", name))?; let mut reader = archive.entry(i).await?; @@ -750,10 +750,10 @@ impl Message { name }; - let mut archive = self.get_webxdc_archive(context).await?; + let archive = self.get_webxdc_archive(context).await?; if name == "index.html" { - if let Ok(bytes) = get_blob(&mut archive, "manifest.toml").await { + if let Ok(bytes) = get_blob(&archive, "manifest.toml").await { if let Ok(manifest) = parse_webxdc_manifest(&bytes) { if let Some(min_api) = manifest.min_api { if min_api > WEBXDC_API_VERSION { @@ -766,15 +766,15 @@ impl Message { } } - get_blob(&mut archive, name).await + get_blob(&archive, name).await } /// Return info from manifest.toml or from fallbacks. pub async fn get_webxdc_info(&self, context: &Context) -> Result { ensure!(self.viewtype == Viewtype::Webxdc, "No webxdc instance."); - let mut archive = self.get_webxdc_archive(context).await?; + let archive = self.get_webxdc_archive(context).await?; - let mut manifest = get_blob(&mut archive, "manifest.toml") + let mut manifest = get_blob(&archive, "manifest.toml") .await .map(|bytes| parse_webxdc_manifest(&bytes).unwrap_or_default()) .unwrap_or_default(); diff --git a/test-data/image/logo.gif b/test-data/image/logo.gif new file mode 100644 index 000000000..f25b2d0ed Binary files /dev/null and b/test-data/image/logo.gif differ diff --git a/test-data/image/logo.png b/test-data/image/logo.png new file mode 100644 index 000000000..ecc9e9054 Binary files /dev/null and b/test-data/image/logo.png differ