feat: Deduplicate in more places (#6464)

Deduplicate:
- In the REPL
- In `store_from_base64()`, which writes avatars received in headers
- In a few tests
- The saved messages, broadcast, device, archive icons
- The autocrypt setup message

1-2 more PRs, and we can get rid of `BlobObject::create`,
`sanitise_name()`, and some others

deltachat-ffi/deltachat.h

@@ -4752,6 +4752,7 @@ void            dc_msg_set_override_sender_name(dc_msg_t* msg, const char* name)
  * @param file If the message object is used in dc_send_msg() later,
  *     this must be the full path of the image file to send.
  * @param filemime The MIME type of the file. NULL if you don't know or don't care.
+ * @deprecated 2025-01-21 Use dc_msg_set_file_and_deduplicate instead
  */
 void            dc_msg_set_file               (dc_msg_t* msg, const char* file, const char* filemime);
 

deltachat-repl/src/cmdline.rs

@@ -939,7 +939,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             } else {
                 Viewtype::File
             });
-            msg.set_file(arg1, None);
+            msg.set_file_and_deduplicate(&context, Path::new(arg1), None, None)?;
             msg.set_text(arg2.to_string());
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }

src/blob.rs

@@ -439,25 +439,21 @@ impl<'a> BlobObject<'a> {
 
     /// Returns path to the stored Base64-decoded blob.
     ///
-    /// If `data` represents an image of known format, this adds the corresponding extension to
+    /// If `data` represents an image of known format, this adds the corresponding extension.
-    /// `suggested_file_stem`.
+    ///
-    pub(crate) async fn store_from_base64(
+    /// Even though this function is not async, it's OK to call it from an async context.
-        context: &Context,
+    pub(crate) fn store_from_base64(context: &Context, data: &str) -> Result<String> {
-        data: &str,
-        suggested_file_stem: &str,
-    ) -> Result<String> {
         let buf = base64::engine::general_purpose::STANDARD.decode(data)?;
-        let ext = if let Ok(format) = image::guess_format(&buf) {
+        let name = if let Ok(format) = image::guess_format(&buf) {
             if let Some(ext) = format.extensions_str().first() {
-                format!(".{ext}")
+                format!("file.{ext}")
             } else {
                 String::new()
             }
         } else {
             String::new()
         };
-        let blob =
+        let blob = BlobObject::create_and_deduplicate_from_bytes(context, &buf, &name)?;
-            BlobObject::create(context, &format!("{suggested_file_stem}{ext}"), &buf).await?;
         Ok(blob.as_name().to_string())
     }
 
@@ -930,15 +926,18 @@ mod tests {
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_file_name() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let blob = BlobObject::create_and_deduplicate_from_bytes(&t, b"hello", "foo.txt").unwrap();
-        assert_eq!(blob.as_file_name(), "foo.txt");
+        assert_eq!(blob.as_file_name(), "ea8f163db38682925e4491c5e58d4bb.txt");
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_as_rel_path() {
         let t = TestContext::new().await;
-        let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
+        let blob = BlobObject::create_and_deduplicate_from_bytes(&t, b"hello", "foo.txt").unwrap();
-        assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
+        assert_eq!(
+            blob.as_rel_path(),
+            Path::new("ea8f163db38682925e4491c5e58d4bb.txt")
+        );
     }
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -1620,7 +1619,7 @@ mod tests {
             .await
             .context("failed to write file")?;
         let mut msg = Message::new(Viewtype::Sticker);
-        msg.set_file(file.to_str().unwrap(), None);
+        msg.set_file_and_deduplicate(alice, &file, None, None)?;
         let chat = alice.get_self_chat().await;
         let sent = alice.send_msg(chat.id, &mut msg).await;
         let msg = Message::load_from_db(alice, sent.sender_msg_id).await?;

src/chat.rs

@@ -2461,7 +2461,8 @@ pub(crate) async fn update_saved_messages_icon(context: &Context) -> Result<()>
         ChatIdBlocked::lookup_by_contact(context, ContactId::SELF).await?
     {
         let icon = include_bytes!("../assets/icon-saved-messages.png");
-        let blob = BlobObject::create(context, "icon-saved-messages.png", icon).await?;
+        let blob =
+            BlobObject::create_and_deduplicate_from_bytes(context, icon, "saved-messages.png")?;
         let icon = blob.as_name().to_string();
 
         let mut chat = Chat::load_from_db(context, chat_id).await?;
@@ -2476,7 +2477,7 @@ pub(crate) async fn update_device_icon(context: &Context) -> Result<()> {
         ChatIdBlocked::lookup_by_contact(context, ContactId::DEVICE).await?
     {
         let icon = include_bytes!("../assets/icon-device.png");
-        let blob = BlobObject::create(context, "icon-device.png", icon).await?;
+        let blob = BlobObject::create_and_deduplicate_from_bytes(context, icon, "device.png")?;
         let icon = blob.as_name().to_string();
 
         let mut chat = Chat::load_from_db(context, chat_id).await?;
@@ -2496,7 +2497,7 @@ pub(crate) async fn get_broadcast_icon(context: &Context) -> Result<String> {
     }
 
     let icon = include_bytes!("../assets/icon-broadcast.png");
-    let blob = BlobObject::create(context, "icon-broadcast.png", icon).await?;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(context, icon, "broadcast.png")?;
     let icon = blob.as_name().to_string();
     context
         .sql
@@ -2511,7 +2512,7 @@ pub(crate) async fn get_archive_icon(context: &Context) -> Result<String> {
     }
 
     let icon = include_bytes!("../assets/icon-archive.png");
-    let blob = BlobObject::create(context, "icon-archive.png", icon).await?;
+    let blob = BlobObject::create_and_deduplicate_from_bytes(context, icon, "archive.png")?;
     let icon = blob.as_name().to_string();
     context
         .sql

src/config.rs

@@ -686,7 +686,7 @@ impl Context {
         let value = match key {
             Config::Selfavatar if value.is_empty() => None,
             Config::Selfavatar => {
-                config_value = BlobObject::store_from_base64(self, value, "avatar").await?;
+                config_value = BlobObject::store_from_base64(self, value)?;
                 Some(config_value.as_str())
             }
             _ => Some(value),
@@ -1143,6 +1143,8 @@ mod tests {
         Ok(())
     }
 
+    const SAVED_MESSAGES_DEDUPLICATED_FILE: &str = "969142cb84015bc135767bc2370934a.png";
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_sync() -> Result<()> {
         let alice0 = TestContext::new_alice().await;
@@ -1217,7 +1219,7 @@ mod tests {
         let self_chat_avatar_path = self_chat.get_profile_image(&alice0).await?.unwrap();
         assert_eq!(
             self_chat_avatar_path,
-            alice0.get_blobdir().join("icon-saved-messages.png")
+            alice0.get_blobdir().join(SAVED_MESSAGES_DEDUPLICATED_FILE)
         );
         assert!(alice1
             .get_config(Config::Selfavatar)

src/contact.rs

@@ -345,7 +345,7 @@ async fn import_vcard_contact(context: &Context, contact: &VcardContact) -> Resu
         return Ok(id);
     }
     let path = match &contact.profile_image {
-        Some(image) => match BlobObject::store_from_base64(context, image, "avatar").await {
+        Some(image) => match BlobObject::store_from_base64(context, image) {
             Err(e) => {
                 warn!(
                     context,

src/imex/key_transfer.rs

@@ -26,12 +26,11 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
     /* this may require a keypair to be created. this may take a second ... */
     let setup_file_content = render_setup_file(context, &setup_code).await?;
     /* encrypting may also take a while ... */
-    let setup_file_blob = BlobObject::create(
+    let setup_file_blob = BlobObject::create_and_deduplicate_from_bytes(
         context,
-        "autocrypt-setup-message.html",
         setup_file_content.as_bytes(),
-    )
+        "autocrypt-setup-message.html",
-    .await?;
+    )?;
 
     let chat_id = ChatId::create_for_contact(context, ContactId::SELF).await?;
     let mut msg = Message {
@@ -39,6 +38,8 @@ pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
         ..Default::default()
     };
     msg.param.set(Param::File, setup_file_blob.as_name());
+    msg.param
+        .set(Param::Filename, "autocrypt-setup-message.html");
     msg.subject = stock_str::ac_setup_msg_subject(context).await;
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");

src/mimeparser.rs

@@ -12,7 +12,6 @@ use deltachat_derive::{FromSql, ToSql};
 use format_flowed::unformat_flowed;
 use lettre_email::mime::Mime;
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
-use rand::distributions::{Alphanumeric, DistString};
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::authres::handle_authres;
@@ -652,17 +651,13 @@ impl MimeMessage {
     }
 
     /// Parses avatar action headers.
-    async fn parse_avatar_headers(&mut self, context: &Context) {
+    fn parse_avatar_headers(&mut self, context: &Context) {
         if let Some(header_value) = self.get_header(HeaderDef::ChatGroupAvatar) {
-            self.group_avatar = self
+            self.group_avatar = self.avatar_action_from_header(context, header_value.to_string());
-                .avatar_action_from_header(context, header_value.to_string())
-                .await;
         }
 
         if let Some(header_value) = self.get_header(HeaderDef::ChatUserAvatar) {
-            self.user_avatar = self
+            self.user_avatar = self.avatar_action_from_header(context, header_value.to_string());
-                .avatar_action_from_header(context, header_value.to_string())
-                .await;
         }
     }
 
@@ -762,7 +757,7 @@ impl MimeMessage {
 
     async fn parse_headers(&mut self, context: &Context) -> Result<()> {
         self.parse_system_message_headers(context);
-        self.parse_avatar_headers(context).await;
+        self.parse_avatar_headers(context);
         self.parse_videochat_headers();
         if self.delivery_report.is_none() {
             self.squash_attachment_parts();
@@ -856,7 +851,7 @@ impl MimeMessage {
         Ok(())
     }
 
-    async fn avatar_action_from_header(
+    fn avatar_action_from_header(
         &mut self,
         context: &Context,
         header_value: String,
@@ -868,15 +863,7 @@ impl MimeMessage {
             .collect::<String>()
             .strip_prefix("base64:")
         {
-            // Add random suffix to the filename
+            match BlobObject::store_from_base64(context, base64) {
-            // to prevent the UI from accidentally using
-            // cached "avatar.jpg".
-            let suffix = Alphanumeric
-                .sample_string(&mut rand::thread_rng(), 7)
-                .to_lowercase();
-
-            match BlobObject::store_from_base64(context, base64, &format!("avatar-{suffix}")).await
-            {
                 Ok(path) => Some(AvatarAction::Change(path)),
                 Err(err) => {
                     warn!(

src/net/http.rs

@@ -13,7 +13,7 @@ use crate::context::Context;
 use crate::net::proxy::ProxyConfig;
 use crate::net::session::SessionStream;
 use crate::net::tls::wrap_rustls;
-use crate::tools::{create_id, time};
+use crate::tools::time;
 
 /// HTTP(S) GET response.
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -119,12 +119,8 @@ fn http_url_cache_timestamps(url: &str, mimetype: Option<&str>) -> (i64, i64) {
 
 /// Places the binary into HTTP cache.
 async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Result<()> {
-    let blob = BlobObject::create(
+    let blob =
-        context,
+        BlobObject::create_and_deduplicate_from_bytes(context, response.blob.as_slice(), "")?;
-        &format!("http_cache_{}", create_id()),
-        response.blob.as_slice(),
-    )
-    .await?;
 
     let (expires, stale) = http_url_cache_timestamps(url, response.mimetype.as_deref());
     context

src/stock_str.rs

@@ -1415,9 +1415,10 @@ impl Context {
         // add welcome-messages. by the label, this is done only once,
         // if the user has deleted the message or the chat, it is not added again.
         let image = include_bytes!("../assets/welcome-image.jpg");
-        let blob = BlobObject::create(self, "welcome-image.jpg", image).await?;
+        let blob = BlobObject::create_and_deduplicate_from_bytes(self, image, "welcome.jpg")?;
         let mut msg = Message::new(Viewtype::Image);
         msg.param.set(Param::File, blob.as_name());
+        msg.param.set(Param::Filename, "welcome-image.jpg");
         chat::add_device_msg(self, Some("core-welcome-image"), Some(&mut msg)).await?;
 
         let mut msg = Message::new_text(welcome_message(self).await);