Compare commits

..

1 Commits

Author SHA1 Message Date
Simon Laux
8e47e50a7e feat: add message parser api to jsonrpc and cffi
api: cffi add `dc_parse_message_text_to_ast_json` and `dc_msg_get_parsed_text_as_json

api: jsonrpc: add `get_parsed_message_text_ast_json` and `parse_text_to_ast_json`
2023-10-10 00:54:58 +00:00
25 changed files with 183 additions and 152 deletions

View File

@@ -1,40 +1,5 @@
# Changelog
## [1.125.0] - 2023-10-14
### API-Changes
- [**breaking**] deltachat-rpc-client: Replace `asyncio` with threads.
- Validate boolean values passed to `set_config`. Attempts to set values other than `0` and `1` will result in an error.
### CI
- Reduce required Python version for deltachat-rpc-client from 3.8 to 3.7.
### Features / Changes
- Add developer option to disable IDLE.
### Fixes
- `deltachat-rpc-client`: Run `deltachat-rpc-server` in its own process group. This prevents reception of `SIGINT` by the server when the bot is terminated with `^C`.
- python: Don't automatically set the displayname to "bot" when setting log level.
- Don't update `timestamp`, `timestamp_rcvd`, `state` when replacing partially downloaded message ([#4700](https://github.com/deltachat/deltachat-core-rust/pull/4700)).
- Assign encrypted partially downloaded group messages to 1:1 chat ([#4757](https://github.com/deltachat/deltachat-core-rust/pull/4757)).
- Return all contacts from `Contact::get_all` for bots ([#4811](https://github.com/deltachat/deltachat-core-rust/pull/4811)).
- Set connectivity status to "connected" during fake idle.
- Return verifier contacts regardless of their origin.
- Don't try to send more MDNs if there's a temporary SMTP error ([#4534](https://github.com/deltachat/deltachat-core-rust/pull/4534)).
### Refactor
- deltachat-rpc-client: Close stdin instead of sending `SIGTERM`.
- deltachat-rpc-client: Remove print() calls. Standard `logging` package is for logging instead.
### Tests
- deltachat-rpc-client: Enable logs in pytest.
## [1.124.1] - 2023-10-05
### Fixes
@@ -2914,4 +2879,3 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
[1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0
[1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0
[1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1
[1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0

29
Cargo.lock generated
View File

@@ -1103,7 +1103,7 @@ dependencies = [
[[package]]
name = "deltachat"
version = "1.125.0"
version = "1.124.1"
dependencies = [
"ansi_term",
"anyhow",
@@ -1118,6 +1118,7 @@ dependencies = [
"chrono",
"criterion",
"deltachat_derive",
"deltachat_message_parser",
"email",
"encoded-words",
"escaper",
@@ -1179,7 +1180,7 @@ dependencies = [
[[package]]
name = "deltachat-jsonrpc"
version = "1.125.0"
version = "1.124.1"
dependencies = [
"anyhow",
"async-channel",
@@ -1203,7 +1204,7 @@ dependencies = [
[[package]]
name = "deltachat-repl"
version = "1.125.0"
version = "1.124.1"
dependencies = [
"ansi_term",
"anyhow",
@@ -1218,7 +1219,7 @@ dependencies = [
[[package]]
name = "deltachat-rpc-server"
version = "1.125.0"
version = "1.124.1"
dependencies = [
"anyhow",
"deltachat",
@@ -1243,7 +1244,7 @@ dependencies = [
[[package]]
name = "deltachat_ffi"
version = "1.125.0"
version = "1.124.1"
dependencies = [
"anyhow",
"deltachat",
@@ -1259,6 +1260,18 @@ dependencies = [
"yerpc",
]
[[package]]
name = "deltachat_message_parser"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826f80f4b32f51457773351b2d821dc1f45273a38235e8fd3bdf662b67b70bcd"
dependencies = [
"nom",
"serde",
"serde_derive",
"unic-idna-punycode",
]
[[package]]
name = "der"
version = "0.6.1"
@@ -5221,6 +5234,12 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
[[package]]
name = "unic-idna-punycode"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06feaedcbf9f1fc259144d833c0d630b8b15207b0486ab817d29258bc89f2f8a"
[[package]]
name = "unicode-bidi"
version = "0.3.13"

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat"
version = "1.125.0"
version = "1.124.1"
edition = "2021"
license = "MPL-2.0"
rust-version = "1.65"
@@ -94,6 +94,7 @@ toml = "0.7"
trust-dns-resolver = "0.22"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
deltachat_message_parser = "0.8.0"
[dev-dependencies]
ansi_term = "0.12.0"
@@ -154,8 +155,4 @@ harness = false
[features]
default = ["vendored"]
internals = []
vendored = [
"async-native-tls/vendored",
"rusqlite/bundled-sqlcipher-vendored-openssl",
"reqwest/native-tls-vendored"
]
vendored = ["async-native-tls/vendored", "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored"]

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat_ffi"
version = "1.125.0"
version = "1.124.1"
description = "Deltachat FFI"
edition = "2018"
readme = "README.md"

View File

@@ -503,9 +503,6 @@ char* dc_get_blobdir (const dc_context_t* context);
* to not mess up with non-delivery-reports or read-receipts.
* 0=no limit (default).
* Changes affect future messages only.
* - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
* seconds. 2 days by default.
* This is not supposed to be changed by UIs and only used for testing.
* - `ui.*` = All keys prefixed by `ui.` can be used by the user-interfaces for system-specific purposes.
* The prefix should be followed by the system and maybe subsystem,
* e.g. `ui.desktop.foo`, `ui.desktop.linux.bar`, `ui.android.foo`, `ui.dc40.bar`, `ui.bot.simplebot.baz`.
@@ -3996,6 +3993,36 @@ int64_t dc_msg_get_sort_timestamp (const dc_msg_t* msg);
char* dc_msg_get_text (const dc_msg_t* msg);
#define MESSAGE_PARSER_MODE_ONLY_TEXT 0x00
#define MESSAGE_PARSER_MODE_DESKTOP_SET 0x01
#define MESSAGE_PARSER_MODE_MARKDOWN 0x02
/**
* Parse text with the message parser.
*
* @memberof dc_context_t
* @param input The text to parse.
* @param mode Sets the parsing mode, you can choose between MESSAGE_PARSER_MODE_ONLY_TEXT, MESSAGE_PARSER_MODE_DESKTOP_SET and MESSAGE_PARSER_MODE_MARKDOWN.
* Look at https://github.com/deltachat/message-parser/blob/master/spec.md#modes-of-the-parser to learn more about the parser modes.
* @return Abstract Syntax Tree for your message that you can use to display parts of a message specially like links.
* This ast is returned in json (look at the sourcecode for reference for the format: https://github.com/deltachat/message-parser/blob/master/src/parser/mod.rs#L11)
*/
char* dc_parse_message_text_to_ast_json (const char* input, int mode);
/**
* Parse the text of a message with the message parser.
*
* @memberof dc_msg_t
* @param msg The message object.
* @param mode Sets the parsing mode, you can choose between MESSAGE_PARSER_MODE_ONLY_TEXT, MESSAGE_PARSER_MODE_DESKTOP_SET and MESSAGE_PARSER_MODE_MARKDOWN.
* Look at https://github.com/deltachat/message-parser/blob/master/spec.md#modes-of-the-parser to learn more about the parser modes.
* @return Abstract Syntax Tree for your message that you can use to display parts of a message specially like links.
* This ast is returned in json (look at the sourcecode for reference for the format: https://github.com/deltachat/message-parser/blob/master/src/parser/mod.rs#L11)
*/
char* dc_msg_get_parsed_text_as_json (const dc_msg_t* msg, int mode);
/**
* Get the subject of the e-mail.
* If there is no subject associated with the message, an empty string is returned.

View File

@@ -31,6 +31,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
use deltachat::imex::BackupProvider;
use deltachat::key::{DcKey, DcSecretKey};
use deltachat::message::MsgId;
use deltachat::message_parser::parser;
use deltachat::net::read_url_blob;
use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
@@ -3356,6 +3357,58 @@ pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_cha
ffi_msg.message.get_text().strdup()
}
#[no_mangle]
pub unsafe extern "C" fn dc_parse_message_text_to_ast_json(
text: *const libc::c_char,
mode: u32,
) -> *mut libc::c_char {
if text.is_null() {
eprintln!("ignoring careless call to dc_parse_message_text_to_ast_json()");
}
let text = to_string_lossy(text);
let result = match mode {
0 /* OnlyText */ => parser::parse_only_text(&text),
1 /* DesktopSet */ => parser::parse_desktop_set(&text),
2 /* Markdown */ => parser::parse_markdown_text(&text),
_ => {
eprintln!("ignoring careless call to dc_parse_message_text_to_ast_json() - invalid mode");
return "".strdup();
}
};
if let Ok(result) = serde_json::to_string(&result) {
result.strdup()
} else {
"".strdup()
}
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_get_parsed_text_as_json(
msg: *mut dc_msg_t,
mode: u32,
) -> *mut libc::c_char {
if msg.is_null() {
eprintln!("ignoring careless call to dc_msg_get_parsed_text_as_json()");
return "".strdup();
}
let ffi_msg = &*msg;
let text = ffi_msg.message.get_text();
let result = match mode {
0 /* OnlyText */ => parser::parse_only_text(&text),
1 /* DesktopSet */ => parser::parse_desktop_set(&text),
2 /* Markdown */ => parser::parse_markdown_text(&text),
_ => {
eprintln!("ignoring careless call to dc_msg_get_parsed_text_as_json() - invalid mode");
return "".strdup();
}
};
if let Ok(result) = serde_json::to_string(&result) {
result.strdup()
} else {
"".strdup()
}
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_get_subject(msg: *mut dc_msg_t) -> *mut libc::c_char {
if msg.is_null() {

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-jsonrpc"
version = "1.125.0"
version = "1.124.1"
description = "DeltaChat JSON-RPC API"
edition = "2021"
default-run = "deltachat-jsonrpc-server"

View File

@@ -43,11 +43,13 @@ use types::contact::ContactObject;
use types::events::Event;
use types::http::HttpResponse;
use types::message::{MessageData, MessageObject, MessageReadReceipt};
use types::message_parser;
use types::provider_info::ProviderInfo;
use types::reactions::JSONRPCReactions;
use types::webxdc::WebxdcMessageInfo;
use self::types::message::MessageLoadResult;
use self::types::message_parser::MessageParserMode;
use self::types::{
chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
location::JsonrpcLocation,
@@ -1081,6 +1083,31 @@ impl CommandApi {
MsgId::new(message_id).get_html(&ctx).await
}
// no specific typings because of https://github.com/dbeckwith/rust-typescript-type-def/issues/18
async fn get_parsed_message_text_ast_json(
&self,
account_id: u32,
message_id: u32,
mode: MessageParserMode,
) -> Result<serde_json::Value> {
let ctx = self.get_context(account_id).await?;
let msg_text = Message::load_from_db(&ctx, MsgId::new(message_id))
.await?
.get_text();
let result = message_parser::parse_text(&msg_text, mode);
Ok(serde_json::to_value(result)?)
}
// no specific typings because of https://github.com/dbeckwith/rust-typescript-type-def/issues/18
async fn parse_text_to_ast_json(
&self,
text: String,
mode: MessageParserMode,
) -> Result<serde_json::Value> {
let result = message_parser::parse_text(&text, mode);
Ok(serde_json::to_value(result)?)
}
/// get multiple messages in one call,
/// if loading one message fails the error is stored in the result object in it's place.
///

View File

@@ -0,0 +1,19 @@
use deltachat::message_parser::parser::{self, Element};
use serde::{Deserialize, Serialize};
use typescript_type_def::TypeDef;
#[repr(u8)]
#[derive(Serialize, Deserialize, TypeDef, schemars::JsonSchema)]
pub enum MessageParserMode {
OnlyText,
DesktopSet,
Markdown,
}
pub fn parse_text(input: &str, mode: MessageParserMode) -> std::vec::Vec<Element> {
match mode {
MessageParserMode::OnlyText => parser::parse_only_text(input),
MessageParserMode::DesktopSet => parser::parse_desktop_set(input),
MessageParserMode::Markdown => parser::parse_markdown_text(input),
}
}

View File

@@ -6,6 +6,7 @@ pub mod events;
pub mod http;
pub mod location;
pub mod message;
pub mod message_parser;
pub mod provider_info;
pub mod qr;
pub mod reactions;

View File

@@ -55,5 +55,5 @@
},
"type": "module",
"types": "dist/deltachat.d.ts",
"version": "1.125.0"
"version": "1.124.1"
}

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-repl"
version = "1.125.0"
version = "1.124.1"
license = "MPL-2.0"
edition = "2021"

View File

@@ -71,6 +71,3 @@ line-length = 120
[tool.isort]
profile = "black"
[tool.pytest.ini_options]
log_cli = true

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-rpc-server"
version = "1.125.0"
version = "1.124.1"
description = "DeltaChat JSON-RPC server"
edition = "2021"
readme = "README.md"

View File

@@ -60,5 +60,5 @@
"test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
},
"types": "node/dist/index.d.ts",
"version": "1.125.0"
"version": "1.124.1"
}

View File

@@ -539,6 +539,8 @@ def test_see_new_verified_member_after_going_online(acfactory, tmp_path, lp):
assert msg_in.text == msg_out.text
assert msg_in.get_sender_contact().addr == ac2_addr
ac1.set_config("bcc_self", "0")
def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
"""Another test for the bug #3836:
@@ -587,67 +589,4 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
assert msg_in.text == msg_out.text
def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
"""Test for the issue #4346:
- User is added to a verified group.
- First device of the user downloads "member added" from the group.
- First device removes "member added" from the server.
- Some new messages are sent to the group.
- Second device comes online, receives these new messages. The result is a verified group with unverified members.
- First device re-gossips Autocrypt keys to the group.
- Now the seconds device has all members verified.
"""
ac1, ac2 = acfactory.get_online_accounts(2)
ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2)
for ac in [ac2, ac2_offl]:
ac.set_config("bcc_self", "1")
ac2.set_config("delete_server_after", "1")
ac2.set_config("gossip_period", "0") # Re-gossip in every message
acfactory.bring_accounts_online()
dir = tmp_path / "exportdir"
dir.mkdir()
ac2.export_self_keys(str(dir))
ac2_offl.import_self_keys(str(dir))
ac2_offl.stop_io()
lp.sec("ac1: create verified-group QR, ac2 scans and joins")
chat1 = ac1.create_group_chat("hello", verified=True)
assert chat1.is_protected()
qr = chat1.get_join_qr()
lp.sec("ac2: start QR-code based join-group protocol")
chat2 = ac2.qr_join_chat(qr)
ac1._evtracker.wait_securejoin_inviter_progress(1000)
# Wait for "Member Me (<addr>) added by <addr>." message.
msg_in = ac2._evtracker.wait_next_incoming_message()
assert msg_in.is_system_message()
lp.sec("ac2: waiting for 'member added' to be deleted on the server")
ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
lp.sec("ac1: sending 'hi' to the group")
ac2.set_config("delete_server_after", "0")
chat1.send_text("hi")
lp.sec("ac2_offl: going online, checking the 'hi' message")
ac2_offl.start_io()
msg_in = ac2_offl._evtracker.wait_next_incoming_message()
assert not msg_in.is_system_message()
assert msg_in.text.startswith("[Sender of this message is not verified:")
ac2_offl_ac1_contact = msg_in.get_sender_contact()
assert ac2_offl_ac1_contact.addr == ac1.get_config("addr")
assert not ac2_offl_ac1_contact.is_verified()
chat2_offl = msg_in.chat
assert chat2_offl.is_protected()
lp.sec("ac2: sending message re-gossiping Autocrypt keys")
chat2.send_text("hi2")
lp.sec("ac2_offl: receiving message")
ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
msg_in = ac2_offl.get_message_by_id(ev.data2)
assert not msg_in.is_system_message()
assert msg_in.text == "hi2"
assert msg_in.chat == chat2_offl
assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
assert ac2_offl_ac1_contact.is_verified()
ac2.set_config("bcc_self", "0")

View File

@@ -1698,10 +1698,12 @@ def test_qr_new_group_unblocked(acfactory, lp):
ac1_new_chat = ac1.create_group_chat("Another group")
ac1_new_chat.add_contact(ac2)
ac1_new_chat.send_text("Hello!")
# Receive "Member added" message.
ac2._evtracker.wait_next_incoming_message()
ac1_new_chat.send_text("Hello!")
# Receive "Hello!" message.
ac2_msg = ac2._evtracker.wait_next_incoming_message()
assert ac2_msg.text == "Hello!"
assert ac2_msg.chat.is_contact_request()

View File

@@ -1 +1 @@
2023-10-14
2023-10-05

View File

@@ -318,13 +318,6 @@ pub enum Config {
/// Last message processed by the bot.
LastMsgId,
/// How often to gossip Autocrypt keys in chats with multiple recipients, in seconds. 2 days by
/// default.
///
/// This is not supposed to be changed by UIs and only used for testing.
#[strum(props(default = "172800"))]
GossipPeriod,
}
impl Context {

View File

@@ -1265,7 +1265,7 @@ impl Contact {
return Ok(Some(ContactId::SELF));
}
match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::Unknown).await? {
match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::AddressBook).await? {
Some(contact_id) => Ok(Some(contact_id)),
None => {
let addr = &self.addr;

View File

@@ -754,6 +754,7 @@ impl Context {
.await?
.to_string(),
);
res.insert(
"debug_logging",
self.get_config_int(Config::DebugLogging).await?.to_string(),
@@ -762,10 +763,6 @@ impl Context {
"last_msg_id",
self.get_config_int(Config::LastMsgId).await?.to_string(),
);
res.insert(
"gossip_period",
self.get_config_int(Config::GossipPeriod).await?.to_string(),
);
let elapsed = self.creation_time.elapsed();
res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));

View File

@@ -391,7 +391,6 @@ impl Imap {
"IMAP-LOGIN as {}",
self.config.lp.user
)));
self.connectivity.set_connected(context).await;
info!(context, "Successfully logged into IMAP server");
Ok(())
}

View File

@@ -110,6 +110,8 @@ pub mod tools;
pub mod accounts;
pub mod reaction;
pub use deltachat_message_parser as message_parser;
/// If set IMAP/incoming and SMTP/outgoing MIME messages will be printed.
pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";

View File

@@ -359,10 +359,9 @@ impl<'a> MimeFactory<'a> {
async fn should_do_gossip(&self, context: &Context) -> Result<bool> {
match &self.loaded {
Loaded::Message { chat } => {
// beside key- and member-changes, force a periodic re-gossip.
// beside key- and member-changes, force re-gossip every 48 hours
let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
let gossip_period = context.get_config_i64(Config::GossipPeriod).await?;
if time() >= gossiped_timestamp + gossip_period {
if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
Ok(true)
} else {
let cmd = self.msg.param.get_cmd();

View File

@@ -673,14 +673,12 @@ pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp)
/// On failure returns an error without removing any `smtp_mdns` entries, the caller is responsible
/// for removing the corresponding entry to prevent endless loop in case the entry is invalid, e.g.
/// points to non-existent message or contact.
///
/// Returns true on success, false on temporary error.
async fn send_mdn_msg_id(
context: &Context,
msg_id: MsgId,
contact_id: ContactId,
smtp: &mut Smtp,
) -> Result<bool> {
) -> Result<()> {
let contact = Contact::get_by_id(context, contact_id).await?;
if contact.is_blocked() {
return Err(format_err!("Contact is blocked"));
@@ -732,14 +730,14 @@ async fn send_mdn_msg_id(
.execute(&q, rusqlite::params_from_iter(additional_msg_ids))
.await?;
}
Ok(true)
Ok(())
}
SendResult::Retry => {
info!(
context,
"Temporary SMTP failure while sending an MDN for {}", msg_id
);
Ok(false)
Ok(())
}
SendResult::Failure(err) => Err(err),
}
@@ -786,17 +784,15 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
.await
.context("failed to update MDN retries count")?;
let res = send_mdn_msg_id(context, msg_id, contact_id, smtp).await;
if let Err(ref err) = res {
if let Err(err) = send_mdn_msg_id(context, msg_id, contact_id, smtp).await {
// If there is an error, for example there is no message corresponding to the msg_id in the
// database, do not try to send this MDN again.
warn!(context, "Error sending MDN for {msg_id}, removing it: {err:#}.");
context
.sql
.execute("DELETE FROM smtp_mdns WHERE msg_id = ?", (msg_id,))
.await?;
Err(err)
} else {
Ok(true)
}
// If there's a temporary error, pretend there are no more MDNs to send. It's unlikely that
// other MDNs could be sent successfully in case of connectivity problems.
res
}