api: add dc_replace_webxdc()

Previously it was required that a directory path is provided to the import API.
Now it is possible to point directly to the .asc file containing a secret key.

This allows UI to present a file selection dialog to the user
and let select the file directly.

Selecting a directory is still supported for backwards compatibility.

deltachat-ffi/deltachat.h

@@ -1175,6 +1175,24 @@ int dc_send_webxdc_status_update (dc_context_t* context, uint32_t msg_id, const
  */
 char* dc_get_webxdc_status_updates (dc_context_t* context, uint32_t msg_id, uint32_t serial);
 
+
+/**
+ * Replaces webxdc app with a new version.
+ *
+ * On the JavaScript side this API could be used like this:
+ * ```
+ * window.webxdc.replaceWebxdc(blob);
+ * ```
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param msg_id The ID of the WebXDC message to be replaced.
+ * @param blob New blob to replace WebXDC with.
+ * @param n Blob size.
+ */
+void dc_replace_webxdc(dc_context_t* context, uint32_t msg_id, uint8_t *blob, size_t n);
+
+
 /**
  * Save a draft for a chat in the database.
  *
@@ -2284,6 +2302,7 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  *
  * - **DC_IMEX_IMPORT_SELF_KEYS** (2) - Import private keys found in the directory given as `param1`.
  *   The last imported key is made the default keys unless its name contains the string `legacy`. Public keys are not imported.
+ *   If `param1` is a filename, import the private key from the file and make it the default.
  *
  * While dc_imex() returns immediately, the started job may take a while,
  * you can stop it using dc_stop_ongoing_process(). During execution of the job,

deltachat-ffi/src/lib.rs

@@ -36,7 +36,7 @@ use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
 use deltachat::stock_str::StockMessage;
 use deltachat::stock_str::StockStrings;
-use deltachat::webxdc::StatusUpdateSerial;
+use deltachat::webxdc::{replace_webxdc, StatusUpdateSerial};
 use deltachat::*;
 use deltachat::{accounts::Accounts, log::LogExt};
 use num_traits::{FromPrimitive, ToPrimitive};
@@ -1097,6 +1097,32 @@ pub unsafe extern "C" fn dc_get_webxdc_status_updates(
     .strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_replace_webxdc(
+    context: *mut dc_context_t,
+    msg_id: u32,
+    blob: *const u8,
+    n: libc::size_t,
+) {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_replace_webxdc()");
+        return;
+    }
+
+    let msg_id = MsgId::new(msg_id);
+    let blob_slice = std::slice::from_raw_parts(blob, n);
+
+    let ctx = &*context;
+
+    block_on(async move {
+        replace_webxdc(ctx, msg_id, blob_slice)
+            .await
+            .context("Failed to replace WebXDC")
+            .log_err(ctx)
+            .ok();
+    })
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_set_draft(
     context: *mut dc_context_t,
@@ -1539,14 +1565,10 @@ pub unsafe extern "C" fn dc_delete_chat(context: *mut dc_context_t, chat_id: u32
     }
     let ctx = &*context;
 
-    block_on(async move {
-        ChatId::new(chat_id)
-            .delete(ctx)
-            .await
-            .context("Failed chat delete")
-            .log_err(ctx)
-            .ok();
-    })
+    block_on(ChatId::new(chat_id).delete(ctx))
+        .context("Failed chat delete")
+        .log_err(ctx)
+        .ok();
 }
 
 #[no_mangle]

deltachat-jsonrpc/src/api/types/message.rs

@@ -318,6 +318,7 @@ pub enum DownloadState {
     Done,
     Available,
     Failure,
+    Undecipherable,
     InProgress,
 }
 
@@ -327,6 +328,7 @@ impl From<download::DownloadState> for DownloadState {
             download::DownloadState::Done => DownloadState::Done,
             download::DownloadState::Available => DownloadState::Available,
             download::DownloadState::Failure => DownloadState::Failure,
+            download::DownloadState::Undecipherable => DownloadState::Undecipherable,
             download::DownloadState::InProgress => DownloadState::InProgress,
         }
     }

deltachat-repl/src/cmdline.rs

@@ -187,6 +187,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         DownloadState::Available => " [⬇ Download available]",
         DownloadState::InProgress => " [⬇ Download in progress...]️",
         DownloadState::Failure => " [⬇ Download failed]",
+        DownloadState::Undecipherable => " [⬇ Decryption failed]",
     };
 
     let temp2 = timestamp_to_str(msg.get_timestamp());

src/chat.rs

@@ -23,6 +23,7 @@ use crate::constants::{
 use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc;
+use crate::download::DownloadState;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::events::EventType;
 use crate::html::new_html_mimepart;
@@ -1038,11 +1039,14 @@ impl ChatId {
         T: Send + 'static,
     {
         let sql = &context.sql;
+        // Do not reply to not fully downloaded messages. Such a message could be a group chat
+        // message that we assigned to 1:1 chat.
         let query = format!(
             "SELECT {fields} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden AND download_state={} \
              ORDER BY timestamp DESC, id DESC \
-             LIMIT 1;"
+             LIMIT 1;",
+             DownloadState::Done as u32,
         );
         let row = sql
             .query_row_optional(
@@ -1067,34 +1071,17 @@ impl ChatId {
         self,
         context: &Context,
     ) -> Result<Option<(String, String, String)>> {
-        if let Some((rfc724_mid, mime_in_reply_to, mime_references, error)) = self
-            .parent_query(
-                context,
-                "rfc724_mid, mime_in_reply_to, mime_references, error",
-                |row: &rusqlite::Row| {
-                    let rfc724_mid: String = row.get(0)?;
-                    let mime_in_reply_to: String = row.get(1)?;
-                    let mime_references: String = row.get(2)?;
-                    let error: String = row.get(3)?;
-                    Ok((rfc724_mid, mime_in_reply_to, mime_references, error))
-                },
-            )
-            .await?
-        {
-            if !error.is_empty() {
-                // Do not reply to error messages.
-                //
-                // An error message could be a group chat message that we failed to decrypt and
-                // assigned to 1:1 chat. A reply to it will show up as a reply to group message
-                // on the other side. To avoid such situations, it is better not to reply to
-                // error messages at all.
-                Ok(None)
-            } else {
-                Ok(Some((rfc724_mid, mime_in_reply_to, mime_references)))
-            }
-        } else {
-            Ok(None)
-        }
+        self.parent_query(
+            context,
+            "rfc724_mid, mime_in_reply_to, mime_references",
+            |row: &rusqlite::Row| {
+                let rfc724_mid: String = row.get(0)?;
+                let mime_in_reply_to: String = row.get(1)?;
+                let mime_references: String = row.get(2)?;
+                Ok((rfc724_mid, mime_in_reply_to, mime_references))
+            },
+        )
+        .await
     }
 
     /// Returns multi-line text summary of encryption preferences of all chat contacts.

src/download.rs

@@ -59,6 +59,9 @@ pub enum DownloadState {
     /// Failed to fully download the message.
     Failure = 20,
 
+    /// Undecipherable message.
+    Undecipherable = 30,
+
     /// Full download of the message is in progress.
     InProgress = 1000,
 }
@@ -80,7 +83,9 @@ impl MsgId {
     pub async fn download_full(self, context: &Context) -> Result<()> {
         let msg = Message::load_from_db(context, self).await?;
         match msg.download_state() {
-            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
+            DownloadState::Done | DownloadState::Undecipherable => {
+                return Err(anyhow!("Nothing to download."))
+            }
             DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
             DownloadState::Available | DownloadState::Failure => {
                 self.update_download_state(context, DownloadState::InProgress)

src/imex.rs

@@ -586,63 +586,74 @@ async fn export_backup_inner(
     Ok(())
 }
 
-/*******************************************************************************
- * Classic key import
- ******************************************************************************/
-async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
-    /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
-    plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
-    Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
+/// Imports secret key from a file.
+async fn import_secret_key(context: &Context, path: &Path, set_default: bool) -> Result<()> {
+    let buf = read_file(context, &path).await?;
+    let armored = std::string::String::from_utf8_lossy(&buf);
+    set_self_key(context, &armored, set_default, false).await?;
+    Ok(())
+}
+
+/// Imports secret keys from the provided file or directory.
+///
+/// If provided path is a file, ASCII-armored secret key is read from the file
+/// and set as the default key.
+///
+/// If provided path is a directory, all files with .asc extension
+/// containing secret keys are imported and the last successfully
+/// imported which does not contain "legacy" in its filename
+/// is set as the default.
+async fn import_self_keys(context: &Context, path: &Path) -> Result<()> {
+    let attr = tokio::fs::metadata(path).await?;
+
+    if attr.is_file() {
+        info!(
+            context,
+            "Importing secret key from {} as the default key.",
+            path.display()
+        );
+        let set_default = true;
+        import_secret_key(context, path, set_default).await?;
+        return Ok(());
+    }
 
-    Maybe we should make the "default" key handlong also a little bit smarter
-    (currently, the last imported key is the standard key unless it contains the string "legacy" in its name) */
-    let mut set_default: bool;
     let mut imported_cnt = 0;
 
-    let dir_name = dir.to_string_lossy();
-    let mut dir_handle = tokio::fs::read_dir(&dir).await?;
+    let mut dir_handle = tokio::fs::read_dir(&path).await?;
     while let Ok(Some(entry)) = dir_handle.next_entry().await {
         let entry_fn = entry.file_name();
         let name_f = entry_fn.to_string_lossy();
-        let path_plus_name = dir.join(&entry_fn);
-        match get_filesuffix_lc(&name_f) {
-            Some(suffix) => {
-                if suffix != "asc" {
-                    continue;
-                }
-                set_default = if name_f.contains("legacy") {
-                    info!(context, "found legacy key '{}'", path_plus_name.display());
-                    false
-                } else {
-                    true
-                }
-            }
-            None => {
+        let path_plus_name = path.join(&entry_fn);
+        if let Some(suffix) = get_filesuffix_lc(&name_f) {
+            if suffix != "asc" {
                 continue;
             }
-        }
+        } else {
+            continue;
+        };
+        let set_default = !name_f.contains("legacy");
         info!(
             context,
-            "considering key file: {}",
+            "Considering key file: {}.",
             path_plus_name.display()
         );
 
-        match read_file(context, &path_plus_name).await {
-            Ok(buf) => {
-                let armored = std::string::String::from_utf8_lossy(&buf);
-                if let Err(err) = set_self_key(context, &armored, set_default, false).await {
-                    info!(context, "set_self_key: {}", err);
-                    continue;
-                }
-            }
-            Err(_) => continue,
+        if let Err(err) = import_secret_key(context, &path_plus_name, set_default).await {
+            warn!(
+                context,
+                "Failed to import secret key from {}: {:#}.",
+                path_plus_name.display(),
+                err
+            );
+            continue;
         }
+
         imported_cnt += 1;
     }
     ensure!(
         imported_cnt > 0,
-        "No private keys found in \"{}\".",
-        dir_name
+        "No private keys found in {}.",
+        path.display()
     );
     Ok(())
 }
@@ -673,7 +684,8 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
         .await?;
 
     for (id, public_key, private_key, is_default) in keys {
-        let id = Some(id).filter(|_| is_default != 0);
+        let id = Some(id).filter(|_| is_default == 0);
+
         if let Ok(key) = public_key {
             if let Err(err) = export_key_to_asc_file(context, dir, id, &key).await {
                 error!(context, "Failed to export public key: {:#}.", err);
@@ -864,14 +876,35 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_key() {
+        let export_dir = tempfile::tempdir().unwrap();
+
         let context = TestContext::new_alice().await;
-        let blobdir = context.ctx.get_blobdir();
-        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(
+            &context.ctx,
+            ImexMode::ExportSelfKeys,
+            export_dir.path(),
+            None,
+        )
+        .await
+        {
             panic!("got error on export: {err:#}");
         }
 
         let context2 = TestContext::new_alice().await;
-        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(
+            &context2.ctx,
+            ImexMode::ImportSelfKeys,
+            export_dir.path(),
+            None,
+        )
+        .await
+        {
+            panic!("got error on import: {err:#}");
+        }
+
+        let keyfile = export_dir.path().join("private-key-default.asc");
+        let context3 = TestContext::new_alice().await;
+        if let Err(err) = imex(&context3.ctx, ImexMode::ImportSelfKeys, &keyfile, None).await {
             panic!("got error on import: {err:#}");
         }
     }

src/mimeparser.rs

@@ -438,6 +438,8 @@ impl MimeMessage {
                         typ: Viewtype::Text,
                         msg_raw: Some(txt.clone()),
                         msg: txt,
+                        // Don't change the error prefix for now,
+                        // receive_imf.rs:lookup_chat_by_reply() checks it.
                         error: Some(format!("Decrypting failed: {err:#}")),
                         ..Default::default()
                     };

src/receive_imf.rs

@@ -1227,6 +1227,8 @@ RETURNING id
                     ephemeral_timestamp,
                     if is_partial_download.is_some() {
                         DownloadState::Available
+                    } else if mime_parser.decrypting_failed {
+                        DownloadState::Undecipherable
                     } else {
                         DownloadState::Done
                     },
@@ -1409,11 +1411,18 @@ async fn lookup_chat_by_reply(
     if let Some(parent) = parent {
         let parent_chat = Chat::load_from_db(context, parent.chat_id).await?;
 
-        if parent.error.is_some() {
-            // If the parent msg is undecipherable, then it may have been assigned to the wrong chat
-            // (undecipherable group msgs often get assigned to the 1:1 chat with the sender).
-            // We don't have any way of finding out whether a msg is undecipherable, so we check for
-            // error.is_some() instead.
+        if parent.download_state != DownloadState::Done
+            // TODO (2023-09-12): Added for backward compatibility with versions that did not have
+            // `DownloadState::Undecipherable`. Remove eventually with the comment in
+            // `MimeMessage::from_bytes()`.
+            || parent
+                .error
+                .as_ref()
+                .filter(|e| e.starts_with("Decrypting failed:"))
+                .is_some()
+        {
+            // If the parent msg is not fully downloaded or undecipherable, it may have been
+            // assigned to the wrong chat (they often get assigned to the 1:1 chat with the sender).
             return Ok(None);
         }
 

src/webxdc.rs

@@ -26,6 +26,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use tokio::io::AsyncReadExt;
 
+use crate::blob::BlobObject;
 use crate::chat::Chat;
 use crate::constants::Chattype;
 use crate::contact::ContactId;
@@ -845,6 +846,35 @@ impl Message {
     }
 }
 
+/// Replaces WebXDC blob of existing message.
+///
+/// This API is supposed to be called from within a WebXDC to replace itself
+/// e.g. with an updated or persistently reconfigured version.
+pub async fn replace_webxdc(context: &Context, msg_id: MsgId, data: &[u8]) -> Result<()> {
+    let mut msg = Message::load_from_db(context, msg_id).await?;
+
+    ensure!(
+        msg.get_viewtype() == Viewtype::Webxdc,
+        "Message {msg_id} is not a WebXDC instance"
+    );
+
+    let blob = BlobObject::create(
+        context,
+        &msg.get_filename()
+            .context("Cannot get filename of exising WebXDC instance")?,
+        data,
+    )
+    .await
+    .context("Failed to create WebXDC replacement blob")?;
+
+    let mut param = msg.param.clone();
+    param.set(Param::File, blob.as_name());
+    msg.param = param;
+    msg.update_param(context).await?;
+
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use serde_json::json;
@@ -2623,4 +2653,62 @@ sth_for_the = "future""#
 
         Ok(())
     }
+
+    /// Tests replacing WebXDC with a newer version.
+    ///
+    /// Updates should be preserved after upgrading.
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_replace_webxdc() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        // Alice sends WebXDC instance.
+        let alice_chat = alice.create_chat(&bob).await;
+        let mut alice_instance = create_webxdc_instance(
+            &alice,
+            "minimal.xdc",
+            include_bytes!("../test-data/webxdc/minimal.xdc"),
+        )
+        .await?;
+        alice_instance.set_text("user added text".to_string());
+        send_msg(&alice, alice_chat.id, &mut alice_instance).await?;
+        let alice_instance = alice.get_last_msg().await;
+        assert_eq!(alice_instance.get_text(), "user added text");
+
+        // Bob receives that instance.
+        let alice_sent_instance = alice.pop_sent_msg().await;
+        let bob_received_instance = bob.recv_msg(&alice_sent_instance).await;
+        assert_eq!(bob_received_instance.get_text(), "user added text");
+
+        // Alice sends WebXDC update.
+        alice
+            .send_webxdc_status_update(alice_instance.id, r#"{"payload": 1}"#, "Alice update")
+            .await?;
+        alice.flush_status_updates().await?;
+        let alice_sent_update = alice.pop_sent_msg().await;
+        bob.recv_msg(&alice_sent_update).await;
+        assert_eq!(
+            bob.get_webxdc_status_updates(bob_received_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":1,"serial":1,"max_serial":1}]"#
+        );
+
+        // Bob replaces WebXDC.
+        replace_webxdc(
+            &bob,
+            bob_received_instance.id,
+            include_bytes!("../test-data/webxdc/with-minimal-manifest.xdc"),
+        )
+        .await
+        .context("Failed to replace WebXDC")?;
+
+        // Updates are not modified.
+        assert_eq!(
+            bob.get_webxdc_status_updates(bob_received_instance.id, StatusUpdateSerial(0))
+                .await?,
+            r#"[{"payload":1,"serial":1,"max_serial":1}]"#
+        );
+
+        Ok(())
+    }
 }