Merge tag 'v1.123.0'

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## [1.123.0] - 2023-09-22
+
+### API-Changes
+
+- Make it possible to import secret key from a file with `DC_IMEX_IMPORT_SELF_KEYS`.
+- [**breaking**] Make `dc_jsonrpc_blocking_call` accept JSON-RPC request.
+
+### Fixes
+
+- `lookup_chat_by_reply()`: Skip not fully downloaded and undecipherable messages ([#4676](https://github.com/deltachat/deltachat-core-rust/pull/4676)).
+- `lookup_chat_by_reply()`: Skip undecipherable parent messages created by older versions ([#4676](https://github.com/deltachat/deltachat-core-rust/pull/4676)).
+- imex: Use "default" in the filename of the default key.
+
+### Miscellaneous Tasks
+
+- Update OpenSSL from 3.1.2 to 3.1.3.
+
 ## [1.122.0] - 2023-09-12
 
 ### API-Changes
@@ -2816,3 +2833,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.120.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.119.1...v1.120.0
 [1.121.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.120.0...v1.121.0
 [1.122.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.121.0...v1.122.0
+[1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0

Cargo.lock

@@ -1085,7 +1085,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.122.0"
+version = "1.123.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1162,7 +1162,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.122.0"
+version = "1.123.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -1186,7 +1186,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "1.122.0"
+version = "1.123.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1201,7 +1201,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.122.0"
+version = "1.123.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1226,7 +1226,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.122.0"
+version = "1.123.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -3057,9 +3057,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "300.1.3+3.1.2"
+version = "300.1.5+3.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd2c101a165fff9935e34def4669595ab1c7847943c42be86e21503e482be107"
+checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491"
 dependencies = [
  "cc",
 ]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.122.0"
+version = "1.123.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.67"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.122.0"
+version = "1.123.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -2273,6 +2273,7 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  *
  * - **DC_IMEX_IMPORT_SELF_KEYS** (2) - Import private keys found in the directory given as `param1`.
  *   The last imported key is made the default keys unless its name contains the string `legacy`. Public keys are not imported.
+ *   If `param1` is a filename, import the private key from the file and make it the default.
  *
  * While dc_imex() returns immediately, the started job may take a while,
  * you can stop it using dc_stop_ongoing_process(). During execution of the job,
@@ -5770,12 +5771,11 @@ char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);
  *
  * @memberof dc_jsonrpc_instance_t
  * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
- * @param method JSON-RPC method name, e.g. `check_email_validity`.
+ * @param input JSON-RPC request.
- * @param params JSON-RPC method parameters, e.g. `["alice@example.org"]`.
  * @return JSON-RPC response as string, must be freed using dc_str_unref() after usage.
- *     On error, NULL is returned.
+ *     If there is no response, NULL is returned.
  */
-char* dc_jsonrpc_blocking_call(dc_jsonrpc_instance_t* jsonrpc_instance, const char *method, const char *params);
+char* dc_jsonrpc_blocking_call(dc_jsonrpc_instance_t* jsonrpc_instance, const char *input);
 
 /**
  * @class dc_event_emitter_t

deltachat-ffi/src/lib.rs

@@ -4986,7 +4986,7 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
 #[cfg(feature = "jsonrpc")]
 mod jsonrpc {
     use deltachat_jsonrpc::api::CommandApi;
-    use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcServer, RpcSession};
+    use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession};
 
     use super::*;
 
@@ -5062,25 +5062,24 @@ mod jsonrpc {
     #[no_mangle]
     pub unsafe extern "C" fn dc_jsonrpc_blocking_call(
         jsonrpc_instance: *mut dc_jsonrpc_instance_t,
-        method: *const libc::c_char,
+        input: *const libc::c_char,
-        params: *const libc::c_char,
     ) -> *mut libc::c_char {
         if jsonrpc_instance.is_null() {
             eprintln!("ignoring careless call to dc_jsonrpc_blocking_call()");
             return ptr::null_mut();
         }
         let api = &*jsonrpc_instance;
-        let method = to_string_lossy(method);
+        let input = to_string_lossy(input);
-        let params = to_string_lossy(params);
+        let res = block_on(api.handle.process_incoming(&input));
-        let params: Option<yerpc::Params> = match serde_json::from_str(&params) {
-            Ok(params) => Some(params),
-            Err(_) => None,
-        };
-        let params = params.map(yerpc::Params::into_value).unwrap_or_default();
-        let res = block_on(api.handle.server().handle_request(method, params));
         match res {
-            Ok(res) => res.to_string().strdup(),
+            Some(message) => {
-            Err(_) => ptr::null_mut(),
+                if let Ok(message) = serde_json::to_string(&message) {
+                    message.strdup()
+                } else {
+                    ptr::null_mut()
+                }
+            }
+            None => ptr::null_mut(),
         }
     }
 }

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.122.0"
+version = "1.123.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/src/api/types/message.rs

@@ -318,6 +318,7 @@ pub enum DownloadState {
     Done,
     Available,
     Failure,
+    Undecipherable,
     InProgress,
 }
 
@@ -327,6 +328,7 @@ impl From<download::DownloadState> for DownloadState {
             download::DownloadState::Done => DownloadState::Done,
             download::DownloadState::Available => DownloadState::Available,
             download::DownloadState::Failure => DownloadState::Failure,
+            download::DownloadState::Undecipherable => DownloadState::Undecipherable,
             download::DownloadState::InProgress => DownloadState::InProgress,
         }
     }

deltachat-jsonrpc/typescript/package.json

@@ -55,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.122.0"
+  "version": "1.123.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.122.0"
+version = "1.123.0"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-repl/src/cmdline.rs

@@ -188,6 +188,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         DownloadState::Available => " [⬇ Download available]",
         DownloadState::InProgress => " [⬇ Download in progress...]️",
         DownloadState::Failure => " [⬇ Download failed]",
+        DownloadState::Undecipherable => " [⬇ Decryption failed]",
     };
 
     let temp2 = timestamp_to_str(msg.get_timestamp());

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.122.0"
+version = "1.123.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.122.0"
+  "version": "1.123.0"
 }

python/tests/test_4_lowlevel.py

@@ -1,3 +1,4 @@
+import json
 from queue import Queue
 
 import deltachat as dc
@@ -227,10 +228,26 @@ def test_jsonrpc_blocking_call(tmp_path):
         lib.dc_accounts_unref,
     )
     jsonrpc = ffi.gc(lib.dc_jsonrpc_init(accounts), lib.dc_jsonrpc_unref)
-    res = from_optional_dc_charpointer(
+    res = json.loads(
-        lib.dc_jsonrpc_blocking_call(jsonrpc, b"check_email_validity", b'["alice@example.org"]'),
+        from_optional_dc_charpointer(
+            lib.dc_jsonrpc_blocking_call(
+                jsonrpc,
+                json.dumps(
+                    {"jsonrpc": "2.0", "method": "check_email_validity", "params": ["alice@example.org"], "id": "123"},
+                ).encode("utf-8"),
+            ),
+        ),
     )
-    assert res == "true"
+    assert res == {"jsonrpc": "2.0", "id": "123", "result": True}
 
-    res = from_optional_dc_charpointer(lib.dc_jsonrpc_blocking_call(jsonrpc, b"check_email_validity", b'["alice"]'))
+    res = json.loads(
-    assert res == "false"
+        from_optional_dc_charpointer(
+            lib.dc_jsonrpc_blocking_call(
+                jsonrpc,
+                json.dumps(
+                    {"jsonrpc": "2.0", "method": "check_email_validity", "params": ["alice"], "id": "456"},
+                ).encode("utf-8"),
+            ),
+        ),
+    )
+    assert res == {"jsonrpc": "2.0", "id": "456", "result": False}

release-date.in

@@ -1 +1 @@
-2023-09-12
+2023-09-22

src/chat.rs

@@ -23,6 +23,7 @@ use crate::constants::{
 use crate::contact::{Contact, ContactId, Origin, VerifiedStatus};
 use crate::context::Context;
 use crate::debug_logging::maybe_set_logging_xdc;
+use crate::download::DownloadState;
 use crate::ephemeral::Timer as EphemeralTimer;
 use crate::events::EventType;
 use crate::html::new_html_mimepart;
@@ -1052,11 +1053,14 @@ impl ChatId {
         T: Send + 'static,
     {
         let sql = &context.sql;
+        // Do not reply to not fully downloaded messages. Such a message could be a group chat
+        // message that we assigned to 1:1 chat.
         let query = format!(
             "SELECT {fields} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden \
+             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?) AND NOT hidden AND download_state={} \
              ORDER BY timestamp DESC, id DESC \
-             LIMIT 1;"
+             LIMIT 1;",
+             DownloadState::Done as u32,
         );
         let row = sql
             .query_row_optional(
@@ -1081,34 +1085,17 @@ impl ChatId {
         self,
         context: &Context,
     ) -> Result<Option<(String, String, String)>> {
-        if let Some((rfc724_mid, mime_in_reply_to, mime_references, error)) = self
+        self.parent_query(
-            .parent_query(
+            context,
-                context,
+            "rfc724_mid, mime_in_reply_to, mime_references",
-                "rfc724_mid, mime_in_reply_to, mime_references, error",
+            |row: &rusqlite::Row| {
-                |row: &rusqlite::Row| {
+                let rfc724_mid: String = row.get(0)?;
-                    let rfc724_mid: String = row.get(0)?;
+                let mime_in_reply_to: String = row.get(1)?;
-                    let mime_in_reply_to: String = row.get(1)?;
+                let mime_references: String = row.get(2)?;
-                    let mime_references: String = row.get(2)?;
+                Ok((rfc724_mid, mime_in_reply_to, mime_references))
-                    let error: String = row.get(3)?;
+            },
-                    Ok((rfc724_mid, mime_in_reply_to, mime_references, error))
+        )
-                },
+        .await
-            )
-            .await?
-        {
-            if !error.is_empty() {
-                // Do not reply to error messages.
-                //
-                // An error message could be a group chat message that we failed to decrypt and
-                // assigned to 1:1 chat. A reply to it will show up as a reply to group message
-                // on the other side. To avoid such situations, it is better not to reply to
-                // error messages at all.
-                Ok(None)
-            } else {
-                Ok(Some((rfc724_mid, mime_in_reply_to, mime_references)))
-            }
-        } else {
-            Ok(None)
-        }
     }
 
     /// Returns multi-line text summary of encryption preferences of all chat contacts.

src/download.rs

@@ -59,6 +59,9 @@ pub enum DownloadState {
     /// Failed to fully download the message.
     Failure = 20,
 
+    /// Undecipherable message.
+    Undecipherable = 30,
+
     /// Full download of the message is in progress.
     InProgress = 1000,
 }
@@ -80,7 +83,9 @@ impl MsgId {
     pub async fn download_full(self, context: &Context) -> Result<()> {
         let msg = Message::load_from_db(context, self).await?;
         match msg.download_state() {
-            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
+            DownloadState::Done | DownloadState::Undecipherable => {
+                return Err(anyhow!("Nothing to download."))
+            }
             DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
             DownloadState::Available | DownloadState::Failure => {
                 self.update_download_state(context, DownloadState::InProgress)

src/imex.rs

@@ -588,63 +588,74 @@ async fn export_backup_inner(
     Ok(())
 }
 
-/*******************************************************************************
+/// Imports secret key from a file.
- * Classic key import
+async fn import_secret_key(context: &Context, path: &Path, set_default: bool) -> Result<()> {
- ******************************************************************************/
+    let buf = read_file(context, &path).await?;
-async fn import_self_keys(context: &Context, dir: &Path) -> Result<()> {
+    let armored = std::string::String::from_utf8_lossy(&buf);
-    /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
+    set_self_key(context, &armored, set_default, false).await?;
-    plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
+    Ok(())
-    Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
+}
+
+/// Imports secret keys from the provided file or directory.
+///
+/// If provided path is a file, ASCII-armored secret key is read from the file
+/// and set as the default key.
+///
+/// If provided path is a directory, all files with .asc extension
+/// containing secret keys are imported and the last successfully
+/// imported which does not contain "legacy" in its filename
+/// is set as the default.
+async fn import_self_keys(context: &Context, path: &Path) -> Result<()> {
+    let attr = tokio::fs::metadata(path).await?;
+
+    if attr.is_file() {
+        info!(
+            context,
+            "Importing secret key from {} as the default key.",
+            path.display()
+        );
+        let set_default = true;
+        import_secret_key(context, path, set_default).await?;
+        return Ok(());
+    }
 
-    Maybe we should make the "default" key handlong also a little bit smarter
-    (currently, the last imported key is the standard key unless it contains the string "legacy" in its name) */
-    let mut set_default: bool;
     let mut imported_cnt = 0;
 
-    let dir_name = dir.to_string_lossy();
+    let mut dir_handle = tokio::fs::read_dir(&path).await?;
-    let mut dir_handle = tokio::fs::read_dir(&dir).await?;
     while let Ok(Some(entry)) = dir_handle.next_entry().await {
         let entry_fn = entry.file_name();
         let name_f = entry_fn.to_string_lossy();
-        let path_plus_name = dir.join(&entry_fn);
+        let path_plus_name = path.join(&entry_fn);
-        match get_filesuffix_lc(&name_f) {
+        if let Some(suffix) = get_filesuffix_lc(&name_f) {
-            Some(suffix) => {
+            if suffix != "asc" {
-                if suffix != "asc" {
-                    continue;
-                }
-                set_default = if name_f.contains("legacy") {
-                    info!(context, "found legacy key '{}'", path_plus_name.display());
-                    false
-                } else {
-                    true
-                }
-            }
-            None => {
                 continue;
             }
-        }
+        } else {
+            continue;
+        };
+        let set_default = !name_f.contains("legacy");
         info!(
             context,
-            "considering key file: {}",
+            "Considering key file: {}.",
             path_plus_name.display()
         );
 
-        match read_file(context, &path_plus_name).await {
+        if let Err(err) = import_secret_key(context, &path_plus_name, set_default).await {
-            Ok(buf) => {
+            warn!(
-                let armored = std::string::String::from_utf8_lossy(&buf);
+                context,
-                if let Err(err) = set_self_key(context, &armored, set_default, false).await {
+                "Failed to import secret key from {}: {:#}.",
-                    info!(context, "set_self_key: {}", err);
+                path_plus_name.display(),
-                    continue;
+                err
-                }
+            );
-            }
+            continue;
-            Err(_) => continue,
         }
+
         imported_cnt += 1;
     }
     ensure!(
         imported_cnt > 0,
-        "No private keys found in \"{}\".",
+        "No private keys found in {}.",
-        dir_name
+        path.display()
     );
     Ok(())
 }
@@ -675,7 +686,8 @@ async fn export_self_keys(context: &Context, dir: &Path) -> Result<()> {
         .await?;
 
     for (id, public_key, private_key, is_default) in keys {
-        let id = Some(id).filter(|_| is_default != 0);
+        let id = Some(id).filter(|_| is_default == 0);
+
         if let Ok(key) = public_key {
             if let Err(err) = export_key_to_asc_file(context, dir, id, &key).await {
                 error!(context, "Failed to export public key: {:#}.", err);
@@ -871,14 +883,35 @@ mod tests {
 
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_export_and_import_key() {
+        let export_dir = tempfile::tempdir().unwrap();
+
         let context = TestContext::new_alice().await;
-        let blobdir = context.ctx.get_blobdir();
+        if let Err(err) = imex(
-        if let Err(err) = imex(&context.ctx, ImexMode::ExportSelfKeys, blobdir, None).await {
+            &context.ctx,
+            ImexMode::ExportSelfKeys,
+            export_dir.path(),
+            None,
+        )
+        .await
+        {
             panic!("got error on export: {err:#}");
         }
 
         let context2 = TestContext::new_alice().await;
-        if let Err(err) = imex(&context2.ctx, ImexMode::ImportSelfKeys, blobdir, None).await {
+        if let Err(err) = imex(
+            &context2.ctx,
+            ImexMode::ImportSelfKeys,
+            export_dir.path(),
+            None,
+        )
+        .await
+        {
+            panic!("got error on import: {err:#}");
+        }
+
+        let keyfile = export_dir.path().join("private-key-default.asc");
+        let context3 = TestContext::new_alice().await;
+        if let Err(err) = imex(&context3.ctx, ImexMode::ImportSelfKeys, &keyfile, None).await {
             panic!("got error on import: {err:#}");
         }
     }

src/mimeparser.rs

@@ -438,6 +438,8 @@ impl MimeMessage {
                         typ: Viewtype::Text,
                         msg_raw: Some(txt.clone()),
                         msg: txt,
+                        // Don't change the error prefix for now,
+                        // receive_imf.rs:lookup_chat_by_reply() checks it.
                         error: Some(format!("Decrypting failed: {err:#}")),
                         ..Default::default()
                     };

src/receive_imf.rs

@@ -1254,6 +1254,8 @@ RETURNING id
                     ephemeral_timestamp,
                     if is_partial_download.is_some() {
                         DownloadState::Available
+                    } else if mime_parser.decrypting_failed {
+                        DownloadState::Undecipherable
                     } else {
                         DownloadState::Done
                     },
@@ -1454,11 +1456,18 @@ async fn lookup_chat_by_reply(
     if let Some(parent) = parent {
         let parent_chat = Chat::load_from_db(context, parent.chat_id).await?;
 
-        if parent.error.is_some() {
+        if parent.download_state != DownloadState::Done
-            // If the parent msg is undecipherable, then it may have been assigned to the wrong chat
+            // TODO (2023-09-12): Added for backward compatibility with versions that did not have
-            // (undecipherable group msgs often get assigned to the 1:1 chat with the sender).
+            // `DownloadState::Undecipherable`. Remove eventually with the comment in
-            // We don't have any way of finding out whether a msg is undecipherable, so we check for
+            // `MimeMessage::from_bytes()`.
-            // error.is_some() instead.
+            || parent
+                .error
+                .as_ref()
+                .filter(|e| e.starts_with("Decrypting failed:"))
+                .is_some()
+        {
+            // If the parent msg is not fully downloaded or undecipherable, it may have been
+            // assigned to the wrong chat (they often get assigned to the 1:1 chat with the sender).
             return Ok(None);
         }