From 0248a36561082543455a427b787348bb081e7faf Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 23 Nov 2022 20:47:49 +0300 Subject: [PATCH 01/23] Release 1.102.0 (#3773) --- CHANGELOG.md | 9 +++++++++ Cargo.lock | 8 ++++---- Cargo.toml | 2 +- deltachat-ffi/Cargo.toml | 2 +- deltachat-jsonrpc/Cargo.toml | 2 +- deltachat-jsonrpc/typescript/package.json | 2 +- deltachat-rpc-server/Cargo.toml | 2 +- package.json | 2 +- 8 files changed, 19 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355a3522b..6764bdcd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ ### Changes +### API-Changes + +### Fixes + + +## 1.102.0 + +### Changes + - If an email has multiple From addresses, handle this as if there was no From address, to prevent from forgery attacks. Also, improve handling of emails with invalid From addresses in general #3667 diff --git a/Cargo.lock b/Cargo.lock index f1882dfd8..8cfc2158d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -826,7 +826,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.101.0" +version = "1.102.0" dependencies = [ "ansi_term", "anyhow", @@ -898,7 +898,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.101.0" +version = "1.102.0" dependencies = [ "anyhow", "async-channel", @@ -920,7 +920,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.101.0" +version = "1.102.0" dependencies = [ "anyhow", "deltachat-jsonrpc", @@ -943,7 +943,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.101.0" +version = "1.102.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index fa9a88b2b..aeef4b40f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.101.0" +version = "1.102.0" authors = ["Delta Chat Developers (ML) "] edition = "2021" license = "MPL-2.0" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 76fd067fe..a4f7bf07f 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.101.0" +version = "1.102.0" description = "Deltachat FFI" authors = ["Delta Chat Developers (ML) "] edition = "2018" diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 9c0e3040a..1677c6c6a 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.101.0" +version = "1.102.0" description = "DeltaChat JSON-RPC API" authors = ["Delta Chat Developers (ML) "] edition = "2021" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 0f305e242..0335544e1 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -48,5 +48,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.101.0" + "version": "1.102.0" } \ No newline at end of file diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 358744746..9ba9547e4 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.101.0" +version = "1.102.0" description = "DeltaChat JSON-RPC server" authors = ["Delta Chat Developers (ML) "] edition = "2021" diff --git a/package.json b/package.json index 351dad9a0..c65517f05 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.101.0" + "version": "1.102.0" } \ No newline at end of file From afb7f897222b5c2a0266ea36f46e79dcbf357066 Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 22 Nov 2022 18:33:38 +0000 Subject: [PATCH 02/23] Do not try to redownload the message in case of any error Since switch to async we don't have spurious "database is busy" errors anymore. Since an error is irrecoverable in most cases, we can skip the message. The cost of this is we may accidentally skip a correct message if I/O fails, but the advantage is that we are guaranteed to never confuse irrecoverable error with recoverable one and get stuck in infinite loop redownloading the same message over and over. --- CHANGELOG.md | 1 + src/authres.rs | 14 +++------- src/decrypt.rs | 6 ++-- src/imap.rs | 6 ++-- src/mimeparser.rs | 68 ++++++++++------------------------------------ src/receive_imf.rs | 22 ++++++--------- 6 files changed, 34 insertions(+), 83 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6764bdcd1..3fa164d69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### API-Changes ### Fixes +- Make sure malformed messsages will never block receiving further messages anymore #3771 ## 1.102.0 diff --git a/src/authres.rs b/src/authres.rs index 5620b1cb4..f9f9d11fb 100644 --- a/src/authres.rs +++ b/src/authres.rs @@ -13,8 +13,6 @@ use once_cell::sync::Lazy; use crate::config::Config; use crate::context::Context; use crate::headerdef::HeaderDef; -use crate::mimeparser; -use crate::mimeparser::ParserErrorExt; use crate::tools::time; use crate::tools::EmailAddress; @@ -32,23 +30,19 @@ pub(crate) async fn handle_authres( mail: &ParsedMail<'_>, from: &str, message_time: i64, -) -> mimeparser::ParserResult { +) -> Result { let from_domain = match EmailAddress::new(from) { Ok(email) => email.domain, Err(e) => { // This email is invalid, but don't return an error, we still want to // add a stub to the database so that it's not downloaded again - return Err(anyhow::format_err!("invalid email {}: {:#}", from, e)).map_err_malformed(); + return Err(anyhow::format_err!("invalid email {}: {:#}", from, e)); } }; let authres = parse_authres_headers(&mail.get_headers(), &from_domain); - update_authservid_candidates(context, &authres) - .await - .map_err_sql()?; - compute_dkim_results(context, authres, &from_domain, message_time) - .await - .map_err_sql() + update_authservid_candidates(context, &authres).await?; + compute_dkim_results(context, authres, &from_domain, message_time).await } #[derive(Default, Debug)] diff --git a/src/decrypt.rs b/src/decrypt.rs index cf3e66b98..e9937390a 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -13,7 +13,6 @@ use crate::context::Context; use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey}; use crate::keyring::Keyring; use crate::log::LogExt; -use crate::mimeparser::{self, ParserErrorExt}; use crate::peerstate::Peerstate; use crate::pgp; @@ -61,7 +60,7 @@ pub(crate) async fn prepare_decryption( mail: &ParsedMail<'_>, from: &str, message_time: i64, -) -> mimeparser::ParserResult { +) -> Result { let autocrypt_header = Aheader::from_headers(from, &mail.headers) .ok_or_log_msg(context, "Failed to parse Autocrypt header") .flatten(); @@ -76,8 +75,7 @@ pub(crate) async fn prepare_decryption( // Disallowing keychanges is disabled for now: true, // dkim_results.allow_keychange, ) - .await - .map_err_sql()?; + .await?; Ok(DecryptionInfo { from: from.to_string(), diff --git a/src/imap.rs b/src/imap.rs index fc540d888..91948e0db 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -1364,7 +1364,9 @@ impl Imap { /// Fetches a list of messages by server UID. /// - /// Returns the last uid fetch successfully and the info about each downloaded message. + /// Returns the last UID fetched successfully and the info about each downloaded message. + /// If the message is incorrect or there is a failure to write a message to the database, + /// it is skipped and the error is logged. pub(crate) async fn fetch_many_msgs( &mut self, context: &Context, @@ -1474,12 +1476,12 @@ impl Imap { if let Some(m) = received_msg { received_msgs.push(m); } - last_uid = Some(server_uid) } Err(err) => { warn!(context, "receive_imf error: {:#}", err); } }; + last_uid = Some(server_uid) } } diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 7dec08f53..24a6941e7 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -157,38 +157,9 @@ impl Default for SystemMessage { const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup"; -#[derive(Debug, thiserror::Error)] -pub(crate) enum ParserError { - #[error("{}", _0)] - Malformed(anyhow::Error), - - #[error("{:#}", _0)] - Sql(anyhow::Error), -} - -pub(crate) type ParserResult = std::result::Result; - -pub(crate) trait ParserErrorExt -where - Self: std::marker::Sized, -{ - fn map_err_malformed(self) -> ParserResult; - fn map_err_sql(self) -> ParserResult; -} - -impl> ParserErrorExt for Result { - fn map_err_malformed(self) -> ParserResult { - self.map_err(|e| ParserError::Malformed(e.into())) - } - - fn map_err_sql(self) -> ParserResult { - self.map_err(|e| ParserError::Sql(e.into())) - } -} - impl MimeMessage { pub async fn from_bytes(context: &Context, body: &[u8]) -> Result { - Ok(MimeMessage::from_bytes_with_partial(context, body, None).await?) + MimeMessage::from_bytes_with_partial(context, body, None).await } /// Parse a mime message. @@ -199,8 +170,8 @@ impl MimeMessage { context: &Context, body: &[u8], partial: Option, - ) -> ParserResult { - let mail = mailparse::parse_mail(body).map_err_malformed()?; + ) -> Result { + let mail = mailparse::parse_mail(body)?; let message_time = mail .headers @@ -227,7 +198,7 @@ impl MimeMessage { ); // Parse hidden headers. - let mimetype = mail.ctype.mimetype.parse::().map_err_malformed()?; + let mimetype = mail.ctype.mimetype.parse::()?; if mimetype.type_() == mime::MULTIPART && mimetype.subtype().as_str() == "mixed" { if let Some(part) = mail.subparts.first() { for field in &part.headers { @@ -245,7 +216,7 @@ impl MimeMessage { headers.remove("secure-join-fingerprint"); headers.remove("chat-verified"); - let from = from.context("No from in message").map_err_malformed()?; + let from = from.context("No from in message")?; let mut decryption_info = prepare_decryption(context, &mail, &from.addr, message_time).await?; @@ -265,7 +236,7 @@ impl MimeMessage { // autocrypt message. mail_raw = raw; - let decrypted_mail = mailparse::parse_mail(&mail_raw).map_err_malformed()?; + let decrypted_mail = mailparse::parse_mail(&mail_raw)?; if std::env::var(crate::DCC_MIME_DEBUG).is_ok() { info!(context, "decrypted message mime-body:"); println!("{}", String::from_utf8_lossy(&mail_raw)); @@ -279,8 +250,7 @@ impl MimeMessage { decrypted_mail.headers.get_all_values("Autocrypt-Gossip"); gossiped_addr = update_gossip_peerstates(context, message_time, &mail, gossip_headers) - .await - .map_err_sql()?; + .await?; } // let known protected headers from the decrypted @@ -333,10 +303,7 @@ impl MimeMessage { // && decryption_info.dkim_results.allow_keychange { peerstate.degrade_encryption(message_time); - peerstate - .save_to_db(&context.sql, false) - .await - .map_err_sql()?; + peerstate.save_to_db(&context.sql, false).await?; } } (Ok(mail), HashSet::new(), false) @@ -380,15 +347,11 @@ impl MimeMessage { Some(org_bytes) => { parser .create_stub_from_partial_download(context, org_bytes) - .await - .map_err_sql()?; + .await?; } None => match mail { Ok(mail) => { - parser - .parse_mime_recursive(context, &mail, false) - .await - .map_err_malformed()?; + parser.parse_mime_recursive(context, &mail, false).await?; } Err(err) => { let msg_body = stock_str::cant_decrypt_msg_body(context).await; @@ -409,7 +372,7 @@ impl MimeMessage { parser.maybe_remove_bad_parts(); parser.maybe_remove_inline_mailinglist_footer(); parser.heuristically_parse_ndn(context).await; - parser.parse_headers(context).await.map_err_malformed()?; + parser.parse_headers(context).await?; // Disallowing keychanges is disabled for now // if !decryption_info.dkim_results.allow_keychange { @@ -427,14 +390,11 @@ impl MimeMessage { parser.decoded_data = mail_raw; } - crate::peerstate::maybe_do_aeap_transition(context, &mut decryption_info, &parser) - .await - .map_err_sql()?; + crate::peerstate::maybe_do_aeap_transition(context, &mut decryption_info, &parser).await?; if let Some(peerstate) = decryption_info.peerstate { peerstate .handle_fingerprint_change(context, message_time) - .await - .map_err_sql()?; + .await?; } Ok(parser) @@ -2196,7 +2156,7 @@ mod tests { let mimeparser = MimeMessage::from_bytes_with_partial(&context.ctx, &raw[..], None).await; - assert!(matches!(mimeparser, Err(ParserError::Malformed(_)))); + assert!(mimeparser.is_err()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/receive_imf.rs b/src/receive_imf.rs index a60ebc339..76f7f1e4a 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -28,7 +28,7 @@ use crate::message::{ self, rfc724_mid_exists, Message, MessageState, MessengerMessage, MsgId, Viewtype, }; use crate::mimeparser::{ - parse_message_ids, AvatarAction, MailinglistType, MimeMessage, ParserError, SystemMessage, + parse_message_ids, AvatarAction, MailinglistType, MimeMessage, SystemMessage, }; use crate::param::{Param, Params}; use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus}; @@ -72,15 +72,13 @@ pub async fn receive_imf( /// Receive a message and add it to the database. /// -/// Returns an error on recoverable errors, e.g. database errors. In this case, -/// message parsing should be retried later. +/// Returns an error on database failure or if the message is broken, +/// e.g. has nonstandard MIME structure. /// -/// If message itself is wrong, logs -/// the error and returns success: -/// - If possible, creates a database entry to prevent the message from being -/// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))` -/// - If the message is so wrong that we didn't even create a database entry, -/// returns `Ok(None)` +/// If possible, creates a database entry to prevent the message from being +/// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`. +/// If the message is so wrong that we didn't even create a database entry, +/// returns `Ok(None)`. /// /// If `is_partial_download` is set, it contains the full message size in bytes. /// Do not confuse that with `replace_partial_download` that will be set when the full message is loaded later. @@ -101,9 +99,8 @@ pub(crate) async fn receive_imf_inner( let mut mime_parser = match MimeMessage::from_bytes_with_partial(context, imf_raw, is_partial_download).await { - Err(ParserError::Malformed(err)) => { + Err(err) => { warn!(context, "receive_imf: can't parse MIME: {}", err); - let msg_ids; if !rfc724_mid.starts_with(GENERATED_PREFIX) { let row_id = context @@ -127,7 +124,6 @@ pub(crate) async fn receive_imf_inner( needs_delete_job: false, })); } - Err(ParserError::Sql(err)) => return Err(err), Ok(mime_parser) => mime_parser, }; @@ -2309,7 +2305,7 @@ mod tests { \n\ hello\x00"; let mimeparser = MimeMessage::from_bytes_with_partial(&context.ctx, &raw[..], None).await; - assert!(matches!(mimeparser, Err(ParserError::Malformed(_)))); + assert!(mimeparser.is_err()); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From 36991b5c8ac9fbde73e787eb7429b6051d03e22e Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 21 Nov 2022 09:05:33 -0300 Subject: [PATCH 03/23] Add Python API to send reactions (#3762) --- python/src/deltachat/events.py | 10 +++++++ python/src/deltachat/hookspec.py | 4 +++ python/src/deltachat/message.py | 12 +++++++++ python/src/deltachat/reactions.py | 43 +++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 python/src/deltachat/reactions.py diff --git a/python/src/deltachat/events.py b/python/src/deltachat/events.py index ccb9f877f..72b84a44a 100644 --- a/python/src/deltachat/events.py +++ b/python/src/deltachat/events.py @@ -192,6 +192,12 @@ class FFIEventTracker: return self.account.get_message_by_id(ev.data2) return None + def wait_next_reactions_changed(self): + """wait for and return next reactions-changed message""" + ev = self.get_matching("DC_EVENT_REACTIONS_CHANGED") + assert ev.data1 > 0 + return self.account.get_message_by_id(ev.data2) + def wait_msg_delivered(self, msg): ev = self.get_matching("DC_EVENT_MSG_DELIVERED") assert ev.data1 == msg.chat.id @@ -296,6 +302,10 @@ class EventThread(threading.Thread): "ac_incoming_message", dict(message=msg), ) + elif name == "DC_EVENT_REACTIONS_CHANGED": + assert ffi_event.data1 > 0 + msg = account.get_message_by_id(ffi_event.data2) + yield "ac_reactions_changed", dict(message=msg) elif name == "DC_EVENT_MSG_DELIVERED": msg = account.get_message_by_id(ffi_event.data2) yield "ac_message_delivered", dict(message=msg) diff --git a/python/src/deltachat/hookspec.py b/python/src/deltachat/hookspec.py index 2a76dafb8..4d2fe6960 100644 --- a/python/src/deltachat/hookspec.py +++ b/python/src/deltachat/hookspec.py @@ -49,6 +49,10 @@ class PerAccount: def ac_outgoing_message(self, message): """Called on each outgoing message (both system and "normal").""" + @account_hookspec + def ac_reactions_changed(self, message): + """Called when message reactions changed.""" + @account_hookspec def ac_message_delivered(self, message): """Called when an outgoing message has been delivered to SMTP. diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index 9ed5b132f..e0b4d468f 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -9,6 +9,7 @@ from typing import Optional, Union from . import const, props from .capi import ffi, lib from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer +from .reactions import Reactions class Message(object): @@ -161,6 +162,17 @@ class Message(object): ) ) + def send_reaction(self, reaction: str): + """Send a reaction to message and return the resulting Message instance.""" + msg_id = lib.dc_send_reaction(self.account._dc_context, self.id, as_dc_charpointer(reaction)) + if msg_id == 0: + raise ValueError("reaction could not be send") + return Message.from_db(self.account, msg_id) + + def get_reactions(self) -> Reactions: + """Get :class:`deltachat.reactions.Reactions` to the message.""" + return Reactions.from_msg(self) + def is_system_message(self): """return True if this message is a system/info message.""" return bool(lib.dc_msg_is_info(self._dc_msg)) diff --git a/python/src/deltachat/reactions.py b/python/src/deltachat/reactions.py new file mode 100644 index 000000000..9e9ed9555 --- /dev/null +++ b/python/src/deltachat/reactions.py @@ -0,0 +1,43 @@ +""" The Reactions object. """ + +from .capi import ffi, lib +from .cutil import from_dc_charpointer, iter_array + + +class Reactions(object): + """Reactions object. + + You obtain instances of it through :class:`deltachat.message.Message`. + """ + + def __init__(self, account, dc_reactions): + assert isinstance(account._dc_context, ffi.CData) + assert isinstance(dc_reactions, ffi.CData) + assert dc_reactions != ffi.NULL + self.account = account + self._dc_reactions = dc_reactions + + def __repr__(self): + return "".format(self._dc_reactions) + + @classmethod + def from_msg(cls, msg): + assert msg.id > 0 + return cls( + msg.account, + ffi.gc(lib.dc_get_msg_reactions(msg.account._dc_context, msg.id), lib.dc_reactions_unref), + ) + + def get_contacts(self) -> list: + """Get list of contacts reacted to the message. + + :returns: list of :class:`deltachat.contact.Contact` objects for this reaction. + """ + from .contact import Contact + + dc_array = ffi.gc(lib.dc_reactions_get_contacts(self._dc_reactions), lib.dc_array_unref) + return list(iter_array(dc_array, lambda x: Contact(self.account, x))) + + def get_by_contact(self, contact) -> str: + """Get a string containing space-separated reactions of a single :class:`deltachat.contact.Contact`.""" + return from_dc_charpointer(lib.dc_reactions_get_by_contact_id(self._dc_reactions, contact.id)) From 0b44886b6285f63c7b450df2f452f72cda4be7bd Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 21 Nov 2022 09:06:23 -0300 Subject: [PATCH 04/23] Add a test for bug "Partially downloaded messages are received out of order" (#3688) + add Message.download_state property. --- python/src/deltachat/_build.py | 1 + python/src/deltachat/message.py | 11 ++++++ python/tests/test_1_online.py | 60 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 09795b3db..995dd34b8 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -156,6 +156,7 @@ def extract_defines(flags): | DC_KEY_GEN | DC_IMEX | DC_CONNECTIVITY + | DC_DOWNLOAD ) # End of prefix matching _[\w_]+ # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048 ) # Close the capturing group, this contains diff --git a/python/src/deltachat/message.py b/python/src/deltachat/message.py index e0b4d468f..0891b46cf 100644 --- a/python/src/deltachat/message.py +++ b/python/src/deltachat/message.py @@ -461,6 +461,17 @@ class Message(object): """mark this message as seen.""" self.account.mark_seen_messages([self.id]) + # + # Message download state + # + @property + def download_state(self): + assert self.id > 0 + + # load message from db to get a fresh/current state + dc_msg = ffi.gc(lib.dc_get_msg(self.account._dc_context, self.id), lib.dc_msg_unref) + return lib.dc_msg_get_download_state(dc_msg) + # some code for handling DC_MSG_* view types diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 2a7f1d78b..85712af80 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1267,6 +1267,66 @@ def test_send_and_receive_image(acfactory, lp, data): assert m == msg_in +def test_reaction_to_partially_fetched_msg(acfactory, lp, tmpdir): + """See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded + messages are received out of order". + + If the Inbox contains X small messages followed by Y large messages followed by Z small + messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages. + + This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced + with online test as follows: + - Bob enables download limit and goes offline. + - Alice sends a large message to Bob and reacts to this message with a thumbs-up. + - Bob goes online + - Bob first processes a reaction message and throws it away because there is no corresponding + message, then processes a partially downloaded message. + - As a result, Bob does not see a reaction + """ + download_limit = 32768 + ac1, ac2 = acfactory.get_online_accounts(2) + ac1_addr = ac1.get_config("addr") + chat = ac1.create_chat(ac2) + ac2.set_config("download_limit", str(download_limit)) + ac2.stop_io() + + reactions_queue = queue.Queue() + + class InPlugin: + @account_hookimpl + def ac_reactions_changed(self, message): + reactions_queue.put(message) + + ac2.add_account_plugin(InPlugin()) + + lp.sec("sending small+large messages from ac1 to ac2") + msgs = [] + msgs.append(chat.send_text("hi")) + path = tmpdir.join("large") + with open(path, "wb") as fout: + fout.write(os.urandom(download_limit + 1)) + msgs.append(chat.send_file(path.strpath)) + + lp.sec("sending a reaction to the large message from ac1 to ac2") + react_str = "\N{THUMBS UP SIGN}" + msgs.append(msgs[-1].send_reaction(react_str)) + + for m in msgs: + ac1._evtracker.wait_msg_delivered(m) + ac2.start_io() + + lp.sec("wait for ac2 to receive a reaction") + msg2 = ac2._evtracker.wait_next_reactions_changed() + assert msg2.get_sender_contact().addr == ac1_addr + assert msg2.download_state == const.DC_DOWNLOAD_AVAILABLE + assert reactions_queue.get() == msg2 + reactions = msg2.get_reactions() + contacts = reactions.get_contacts() + assert len(contacts) == 1 + assert contacts[0].addr == ac1_addr + assert reactions.get_by_contact(contacts[0]) == react_str + + def test_import_export_online_all(acfactory, tmpdir, data, lp): (ac1,) = acfactory.get_online_accounts(1) From 6f0985dcaab4e9a79f42bd1f6435133c5e337d49 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 23 Nov 2022 22:38:12 +0000 Subject: [PATCH 05/23] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fa164d69..5b89e1ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changes ### API-Changes +- Add Python API to send reactions #3762 ### Fixes - Make sure malformed messsages will never block receiving further messages anymore #3771 From 9783da5d8e1d314b1f60721865f922d67196921b Mon Sep 17 00:00:00 2001 From: iequidoo Date: Tue, 22 Nov 2022 16:27:34 -0300 Subject: [PATCH 06/23] receive_imf: trim() "Chat-Group-Name{,-Changed}:" headers content (#3650) It's a w/a for "Space added before long group names after MIME serialization/deserialization" issue. DC itself never creates group names with leading/trailing whitespace, so it can be safely removed. On the sender side there's no trim() because group names anyway go through improve_single_line_input(). And I believe we should send the exact name we have in our db. Also there's no check for leading/trailing whitespace because there may be existing user databases with group names having such whitespaces. --- CHANGELOG.md | 1 + python/tests/test_1_online.py | 23 +++++++++++++++++++++++ src/receive_imf.rs | 13 +++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b89e1ede..08df3e3e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - fix detection of "All mail", "Trash", "Junk" etc folders. #3760 - fetch messages sequentially to fix reactions on partially downloaded messages #3688 - Fix a bug where one malformed message blocked receiving any further messages #3769 +- strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650 ## 1.101.0 diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 85712af80..39cbae48c 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -405,6 +405,29 @@ def test_forward_own_message(acfactory, lp): assert msg_in.is_forwarded() +def test_long_group_name(acfactory, lp): + """See bug https://github.com/deltachat/deltachat-core-rust/issues/3650 "Space added before long + group names after MIME serialization/deserialization". + + When the mailadm bot creates a group with botadmin, the bot creates is as + "pytest-supportuser-282@x.testrun.org support group" (for example). But in the botadmin's + account object, the group chat is called " pytest-supportuser-282@x.testrun.org support group" + (with an additional space character in the beginning). + """ + ac1, ac2 = acfactory.get_online_accounts(2) + + lp.sec("ac1: creating group chat and sending a message") + group_name = "pytest-supportuser-282@x.testrun.org support group" + group = ac1.create_group_chat(group_name) + group.add_contact(ac2) + group.send_text("message") + + # wait for other account to receive + ev = ac2._evtracker.get_matching("DC_EVENT_INCOMING_MSG") + msg_in = ac2.get_message_by_id(ev.data2) + assert msg_in.chat.get_name() == group_name + + def test_send_self_message(acfactory, lp): ac1 = acfactory.new_online_configuring_account(mvbox_move=True) acfactory.bring_accounts_online() diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 76f7f1e4a..60f9a8e03 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1510,7 +1510,10 @@ async fn create_or_lookup_group( let grpname = mime_parser .get_header(HeaderDef::ChatGroupName) - .context("Chat-Group-Name vanished")?; + .context("Chat-Group-Name vanished")? + // W/a for "Space added before long group names after MIME serialization/deserialization + // #3650" issue. DC itself never creates group names with leading/trailing whitespace. + .trim(); let new_chat_id = ChatId::create_multiuser_record( context, Chattype::Group, @@ -1618,9 +1621,15 @@ async fn apply_group_changes( { better_msg = Some(stock_str::msg_add_member(context, &added_member, from_id).await); recreate_member_list = true; - } else if let Some(old_name) = mime_parser.get_header(HeaderDef::ChatGroupNameChanged) { + } else if let Some(old_name) = mime_parser + .get_header(HeaderDef::ChatGroupNameChanged) + // See create_or_lookup_group() for explanation + .map(|s| s.trim()) + { if let Some(grpname) = mime_parser .get_header(HeaderDef::ChatGroupName) + // See create_or_lookup_group() for explanation + .map(|grpname| grpname.trim()) .filter(|grpname| grpname.len() < 200) { if chat_id From a76b01890080f97092f70cc1ca095f05d14e958a Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 24 Nov 2022 11:23:36 +0000 Subject: [PATCH 07/23] Move changelog entry to the correct section --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08df3e3e2..fc69a88eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Fixes - Make sure malformed messsages will never block receiving further messages anymore #3771 +- strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650 ## 1.102.0 @@ -25,7 +26,6 @@ - fix detection of "All mail", "Trash", "Junk" etc folders. #3760 - fetch messages sequentially to fix reactions on partially downloaded messages #3688 - Fix a bug where one malformed message blocked receiving any further messages #3769 -- strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650 ## 1.101.0 From b341cfd4d928f13d517ef21ea63b31e9d3f6102c Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 23 Nov 2022 23:09:03 +0000 Subject: [PATCH 08/23] mimeparser: assume all Thunderbird users prefer encryption Co-Authored-By: missytake --- CHANGELOG.md | 1 + src/decrypt.rs | 11 ++- src/mimeparser.rs | 9 +- src/receive_imf.rs | 17 ++++ .../message/thunderbird_with_autocrypt.eml | 93 +++++++++++++++++++ 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 test-data/message/thunderbird_with_autocrypt.eml diff --git a/CHANGELOG.md b/CHANGELOG.md index fc69a88eb..a1f003c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ ### Fixes - Make sure malformed messsages will never block receiving further messages anymore #3771 - strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650 +- Assume all Thunderbird users prefer encryption #3774 ## 1.102.0 diff --git a/src/decrypt.rs b/src/decrypt.rs index e9937390a..4f76a2786 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -5,7 +5,7 @@ use std::collections::HashSet; use anyhow::{Context as _, Result}; use mailparse::ParsedMail; -use crate::aheader::Aheader; +use crate::aheader::{Aheader, EncryptPreference}; use crate::authres; use crate::authres::handle_authres; use crate::contact::addr_cmp; @@ -60,11 +60,18 @@ pub(crate) async fn prepare_decryption( mail: &ParsedMail<'_>, from: &str, message_time: i64, + is_thunderbird: bool, ) -> Result { - let autocrypt_header = Aheader::from_headers(from, &mail.headers) + let mut autocrypt_header = Aheader::from_headers(from, &mail.headers) .ok_or_log_msg(context, "Failed to parse Autocrypt header") .flatten(); + if is_thunderbird { + if let Some(autocrypt_header) = &mut autocrypt_header { + autocrypt_header.prefer_encrypt = EncryptPreference::Mutual; + } + } + let dkim_results = handle_authres(context, mail, from, message_time).await?; let peerstate = get_autocrypt_peerstate( diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 24a6941e7..064701eb1 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -216,9 +216,16 @@ impl MimeMessage { headers.remove("secure-join-fingerprint"); headers.remove("chat-verified"); + let is_thunderbird = if let Some(user_agent) = headers.get("user-agent") { + info!(context, "Detected thunderbird"); + user_agent.contains("Thunderbird") + } else { + false + }; + let from = from.context("No from in message")?; let mut decryption_info = - prepare_decryption(context, &mail, &from.addr, message_time).await?; + prepare_decryption(context, &mail, &from.addr, message_time, is_thunderbird).await?; // Memory location for a possible decrypted message. let mut mail_raw = Vec::new(); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 60f9a8e03..13241072d 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2277,6 +2277,7 @@ mod tests { use super::*; + use crate::aheader::EncryptPreference; use crate::chat::get_chat_contacts; use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility}; use crate::chatlist::Chatlist; @@ -5299,4 +5300,20 @@ Reply from different address Ok(()) } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_thunderbird_autocrypt() -> Result<()> { + let t = TestContext::new_bob().await; + t.set_config(Config::ShowEmails, Some("2")).await?; + + let raw = include_bytes!("../test-data/message/thunderbird_with_autocrypt.eml"); + receive_imf(&t, raw, false).await?; + + let peerstate = Peerstate::from_addr(&t, "alice@example.org") + .await? + .unwrap(); + assert_eq!(peerstate.prefer_encrypt, EncryptPreference::Mutual); + + Ok(()) + } } diff --git a/test-data/message/thunderbird_with_autocrypt.eml b/test-data/message/thunderbird_with_autocrypt.eml new file mode 100644 index 000000000..ef9ec15c8 --- /dev/null +++ b/test-data/message/thunderbird_with_autocrypt.eml @@ -0,0 +1,93 @@ +From - Thu, 24 Nov 2022 19:06:16 GMT +X-Mozilla-Status: 0001 +X-Mozilla-Status2: 00800000 +Message-ID: <0bb9ffe1-2596-d997-95b4-1fef8cc4808e@example.org> +Date: Thu, 24 Nov 2022 20:05:57 +0100 +MIME-Version: 1.0 +User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 + Thunderbird/102.4.2 +From: Alice +To: bob@example.net +Content-Language: en-US +Autocrypt: addr=alice@example.org; keydata= + xjMEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5DN + GUFsaWNlIDxhbGljZUBleGFtcGxlLm9yZz7CkAQTFggAOBYhBC5vossjtTLXKGNLWGSwj2Gp + 7ZRDBQJeWHXdAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEGSwj2Gp7ZRDE3oA/i4M + CyDMTsjWqDZoQwX/A/GoTO2/V0wKPhjJJy/8m2pMAPkBjOnGOtx2SZpQvJGTa9h804RY6iDr + RuI8A/8tEEXAA844BF5Ydd0SCisGAQQBl1UBBQEBB0AG7cjWy2SFAU8KnltlubVW67rFiyfp + 01JrRe6Xqy22HQMBCAfCeAQYFggAIBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsM + AAoJEGSwj2Gp7ZRDLo8BAObE8GnsGVwKzNqCvHeWgJsqhjS3C6gvSlV3tEm9XmF6AQDXucIy + VfoBwoyMh2h6cSn/ATn5QJb35pgo+ivp3jsMAg== +Subject: ... +Content-Type: multipart/encrypted; + protocol="application/pgp-encrypted"; + boundary="------------EOdOT2kJUL5hgCilmIhYyVZg" + +This is an OpenPGP/MIME encrypted message (RFC 4880 and 3156) +--------------EOdOT2kJUL5hgCilmIhYyVZg +Content-Type: application/pgp-encrypted +Content-Description: PGP/MIME version identification + +Version: 1 + +--------------EOdOT2kJUL5hgCilmIhYyVZg +Content-Type: application/octet-stream; name="encrypted.asc" +Content-Description: OpenPGP encrypted message +Content-Disposition: inline; filename="encrypted.asc" + +-----BEGIN PGP MESSAGE----- + +wV4D5tq63hTeebASAQdA1dVUsUjGZCOIfCnYtVdmOvKs/BNovI3sG8w1IH4ymTMwAZzgwVbGS5KL ++e1VTD5mUTeVSEYe1cd3VozH4KbNJa1tBlcO0nzGwCPpsTVDMoxIwcBMA+PY3JvEjuMiAQf/d2yj +t0+GyaptwX26bgSqo6vj21W8mcWS5vXOi8wjGwRbPaKKjS4kq1xDOz04eHrE8HUPD8otcXoI8CLz +etJpRbFs0XJP4Cozbsr72dgoWhozRg/iSpBndxWOddTl7Yqo8m/fyhU5uzKZ41m2T8mha6KkKWD8 +QecGdOgieYBucNBjHwWc71p9G6jTnzfy4S4GtGS2gwOSMxpwO7HxpKzsHI4POqFSQbxrl/YRwWSC +f5WqyYcerasIiR/fnOIw8lnvCeQ5rB90eGEDR70YFGt0t4rFBjfGrSPUiWYaTaC1Zvpd+t5sy7zy +FpsS2/aTkwP/UpGqmtFaD/brSouRf9hijNLI0QFTaVmSoI3BKzF8B4zwvtEbOLZjyDb+Va/fZJ3w +nYd2Q/5PPPL+pE4pWKN+jl0TZNzAaqBgvggXomgUqQ7QiksUzym+yuFKrJX0RF2awdrgjQIxjnda +Qp3UFphnFTyYUJpIU9iewjOfVxgPzv7PyuCHYwoP3kh7MJZ6bgbDmOkeFSnjEDJpdf1m9xC9LlBL +beC8scmPs6kx9GARBYSHvyPQ025gN3+XEHh4OrTxHZ91U3IlTfd2kACwOOAXEuhItSHmcNOV0K4M +nI2PH6gW8HgBkWlAPm40K4jUyo3nl1usDiI6ouvYqvW7YUc2hTtPTej1l2/mS57tTt+PFurKs555 +5R9DD/xg9Nx7OuQKy5bIdlXM20UmwuZTOhRJ5kpHFRzLxaHDbSzW+orhRW4llJSevBSAH3cLOjIQ +gh87j+MxG9j0TD2K2A0rcUcxdrnflw+mxcDVaL4payeqmOa+bJyhlftTqH+vqq5DhR68rX5VW+z7 +riqH3o8VbvO2y0XSpYHf1jowkfJj3vr8pynAUIv1dbylUSF5wtrHvzWOprw4bNrdtwQNRNy+JcVF +dUKeNmHaL6XOe4LUWpiI11beRyCpAG52khMCEAO3Q6+4e24cEipbu6suSOtv3OpYDZeHjwNrQIhi +rJg7i9TpMqwOeCvFWK+9UZ+P2n6h9g0/JO2+I82BFGUjVa5IvCTNOgv01GqxWY9ecdtaJjTc+dF2 +OAcRoKwvmtMJlxKEEgveui3BvPA4tuNdSrcoZBrQeo0ZHWVugXPvEZnwfZMcqwwPA+a/sUbZFg0P +Pr0AR0ZHpytnQE9OXE8wEUgT8H1yofQ+5QoZdgMpeAb8zGs+RuviLxcDkb9NtXUAiQ49ooWuFP3L +K9wMlaoWFTq7R+n5JVuSEYRCHC0l0bCV1/+awalT7XltXVCupI4lWzjYs52FZGGzuHG7S50Eufad +m4CQTPVgVaVn8WW2dmpMR8Gj8WbbZdyv21wMGOWjfgT0u3oiDnddGrFOoMNnZHch6rN3FRppoh7h +0U0fi8xxU1+EhUKq+fSIxZNr2iWN2if3Pipbxi9tyK9M41Y6aVF3HWjD58/OEql3aZjJZ1bqpXcE +qsPeFoXX78+7mTDvL75olMk2s/mg4mLqAAWQvTuoiOmj+SgMIFuTtFR+4r/TIFNdamz6AQ3RcmWG +ZcdRii+V27dtMA836vlAwxXRmJyE1LCL1kvUTq+J+AVsZi3xmBLFNlKPTlxswu7vSBrP1DlYOaBq +AgA0lKnkQdeXyDk/VdbTml7ywMW1g6HkFSqKGW/IIAObmBumBcIyHE6dWEHumRQomlJssIlEFSe+ +XEQ0rwedLetJXi5A0AXT1we1wvaKCEg0Pb0ZUxygwNPDrj6MmdodH7gDfyx0mW/7mEMCtIJb5MB+ +TRGPEa/vqdJb8uGtNXUy9UlwMhJ3tYoT7NXY4+IlNjbDH/yleMdwtWP2H2WH8oC+ysXPYXjlT8eU +poxRfJzPMVUn5SA3cvdGXDJWdX8U91j5sf9wuoYE5RBVrrJif3D3l0FpMrlWWoGw7wtZbMC2FaeT +QvdMS5c54IoXBtBTM+/AsTAw7WEE1QSmaQGHnh6xLL5Ns8olsWeKOMlVXdO9jSDbjOGBLr7mWukW +YzLXkH3TtJPQcbVN79af3YPhaHdMYITVKIwfg+vxZlLFHWLJQnkTl+9Qi7u2gKqkNeU7Zqs4E3CR +9K4dHrJMyAZLZ2HA1XQEj0/tMnbTpAzZhj02JRcFobLXK9SQfw7dzGZwMRky8cHcBHoK14P5RIEV +hr+38HSBM6wXtge5gL6DomAACvuORQO4X9x/CTjRt/J8uN3lKK5p+wi3ULeb319CEWiCiqmC1M+C +TADUhPUhUmTinSAVkTEn+BdbH/97dVaJnvd6HtLmdSlw4xqdWUfVL9Qd7+/5L6iwlOzGLKRv97c/ +gCRw+hzXyAom+5C18slSwanMuyPgIyrrFy/kp9Romk9SQr/c0CUF2am99t8G5qvVi/TiJGHyKEXD +aUYd4V7lqNlHMiiasvFHeq8blwmFr7rGEvbZzLNplc6sRUVlYhY2unRfyWsq9mqk3NDRW12Fa0J2 +YxQJlnXHQhNE8EyM/zsD9jCVNwsRZJ9/e5KS+ignmu6gKIR+ItDTwRfNI+NG/YmTgENUTyuO+vQC +CUKS3PCwpP+OEC966ARl7OCMdfn1hEyiAxsZnp1RmFngR6FM+mlGgfUoWNoHvnR1/YyQ4F4dadiA +QINwuSm5faw75F1EeL8Qi+LHKuqt05Pi/V9GJ6TzIkIsEbyyJ5sKHrp4QsU4C1p7ZhPjddz8De8k +6ZdwMIeXxi27WKtsFLcr8JKOBe0imIilKdMBOPS31pc1iJe4472WbWM0aBwdEYmnz9+xfOqnjHtO +0XTMjff7pzV6Y7t/u8J/zm3JS3ykote9HNRQvhZZNeVClVWd0fYFzat5ESnTojZTwHcc/BFTPnhz +VgLyw1KEIy2r3ZyGHu1b8GSYivzl33MOK/NVBQPZUIEfdcQ5vhkAvj+Yx340IYykRFEChwioprXD +LrIbTou7TNT5fTFA+beidHFsL+OE002/LMs6C3erSUW5C/LNjAQMS7cAV2yCyjX+/2GBmmDqnC4r +Ja2x5yik+fbOUPh3kk/md1YvrodlX/JkQeoWRrrVJsX2dr3BgivPJavaN0Jz1eHyxAYKNqlrfd1T +YWEDIisWerTxAVY/rEruZ6+OqLqOtZtn+4SOajOq8KFusglaMZqoYuM+LhPZck9PlZXwRqX08Vlv +8jX5V75BFWRhFd5/LYbnQHI6ZW80Wb2sBNngLL2QJT9yXGCDJb5qCdFwGd3i655pvRJXabeyCtDD +7I2PJcYRDd4stdq07BHyHJmye6vas8mG5QUygyWyUQv78za0m4gLMrRZBgoBDcVpWJUc+cPXzzfG +7PvLZu/Y0SaD5hqTp0LBB1PFxTpzdVeJ21gzVNQ6D4XGLTtdv4K4fOEYoeKEuzGoBaUDtIqz47gd +5rwfQ3ps2slkxfbtQcdKEACKvsCwzqHlgwsxD8QNOFzXYLiiiJBX22fIRoiJeSDMKSZyuFtpykCm +7bOpybPSHv3E7EIr8sIOr9MOe/R5HSthU2IgW1L5Ynr2t9HUnCA8CenkzIQjg0h5sruxcGWCYLx7 +q0f1AQs4Z7SebVbq1SCWVJNX/vc1bVjnjYfri7RX5WMmjJkuSnuIoP6a42cqJcAg7m0STB0elFAy +oO4vW9/JEmFUqLyQmWnoLJHX3IKtWa9CPvE= +=OA6b +-----END PGP MESSAGE----- + +--------------EOdOT2kJUL5hgCilmIhYyVZg-- From 08de326930d7bb827314f92ddee6da46bfb09220 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sun, 27 Nov 2022 10:01:25 +0000 Subject: [PATCH 09/23] Fix misplaced info! message --- src/mimeparser.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 064701eb1..04a4b4d52 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -216,12 +216,12 @@ impl MimeMessage { headers.remove("secure-join-fingerprint"); headers.remove("chat-verified"); - let is_thunderbird = if let Some(user_agent) = headers.get("user-agent") { - info!(context, "Detected thunderbird"); - user_agent.contains("Thunderbird") - } else { - false - }; + let is_thunderbird = headers + .get("user-agent") + .map_or(false, |user_agent| user_agent.contains("Thunderbird")); + if is_thunderbird { + info!(context, "Detected Thunderbird"); + } let from = from.context("No from in message")?; let mut decryption_info = From 2ae9165bfbe6f81079d9f8ed92116745b9454e3b Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 25 Nov 2022 21:06:14 +0000 Subject: [PATCH 10/23] Remove different states of ToSave in peerstate --- src/e2ee.rs | 3 +-- src/peerstate.rs | 61 +++++++++++++++-------------------------------- src/qr.rs | 3 +-- src/securejoin.rs | 6 ++--- src/tests/aeap.rs | 2 +- 5 files changed, 25 insertions(+), 50 deletions(-) diff --git a/src/e2ee.rs b/src/e2ee.rs index cd92c5a3d..f8fe11453 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -147,7 +147,6 @@ mod tests { use crate::chat; use crate::message::{Message, Viewtype}; use crate::param::Param; - use crate::peerstate::ToSave; use crate::test_utils::{bob_keypair, TestContext}; use super::*; @@ -297,7 +296,7 @@ Sent with my Delta Chat Messenger: https://delta.chat"; gossip_key_fingerprint: Some(pub_key.fingerprint()), verified_key: Some(pub_key.clone()), verified_key_fingerprint: Some(pub_key.fingerprint()), - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; vec![(Some(peerstate), addr)] diff --git a/src/peerstate.rs b/src/peerstate.rs index 165fd05fc..09453cac7 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -46,7 +46,7 @@ pub struct Peerstate { pub gossip_key_fingerprint: Option, pub verified_key: Option, pub verified_key_fingerprint: Option, - pub to_save: Option, + pub to_save: bool, pub fingerprint_changed: bool, } @@ -90,13 +90,6 @@ impl fmt::Debug for Peerstate { } } -#[derive(Debug, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)] -#[repr(u8)] -pub enum ToSave { - Timestamps = 0x01, - All = 0x02, -} - impl Peerstate { pub fn from_header(header: &Aheader, message_time: i64) -> Self { Peerstate { @@ -111,7 +104,7 @@ impl Peerstate { gossip_timestamp: 0, verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, } } @@ -137,7 +130,7 @@ impl Peerstate { gossip_timestamp: message_time, verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, } } @@ -229,7 +222,7 @@ impl Peerstate { .map(|s| s.parse::()) .transpose() .unwrap_or_default(), - to_save: None, + to_save: false, fingerprint_changed: false, }; @@ -252,7 +245,7 @@ impl Peerstate { || self.public_key_fingerprint.is_none() || old_public_fingerprint != self.public_key_fingerprint { - self.to_save = Some(ToSave::All); + self.to_save = true; if old_public_fingerprint.is_some() { self.fingerprint_changed = true; } @@ -267,7 +260,7 @@ impl Peerstate { || self.gossip_key_fingerprint.is_none() || old_gossip_fingerprint != self.gossip_key_fingerprint { - self.to_save = Some(ToSave::All); + self.to_save = true; // Warn about gossip key change only if there is no public key obtained from // Autocrypt header, which overrides gossip key. @@ -281,7 +274,7 @@ impl Peerstate { pub fn degrade_encryption(&mut self, message_time: i64) { self.prefer_encrypt = EncryptPreference::Reset; self.last_seen = message_time; - self.to_save = Some(ToSave::All); + self.to_save = true; } pub fn apply_header(&mut self, header: &Aheader, message_time: i64) { @@ -292,19 +285,17 @@ impl Peerstate { if message_time > self.last_seen { self.last_seen = message_time; self.last_seen_autocrypt = message_time; - self.to_save = Some(ToSave::Timestamps); + self.to_save = true; if (header.prefer_encrypt == EncryptPreference::Mutual || header.prefer_encrypt == EncryptPreference::NoPreference) && header.prefer_encrypt != self.prefer_encrypt { self.prefer_encrypt = header.prefer_encrypt; - self.to_save = Some(ToSave::All) } if self.public_key.as_ref() != Some(&header.public_key) { self.public_key = Some(header.public_key.clone()); self.recalc_fingerprint(); - self.to_save = Some(ToSave::All); } } } @@ -316,11 +307,10 @@ impl Peerstate { if message_time > self.gossip_timestamp { self.gossip_timestamp = message_time; - self.to_save = Some(ToSave::Timestamps); + self.to_save = true; if self.gossip_key.as_ref() != Some(&gossip_header.public_key) { self.gossip_key = Some(gossip_header.public_key.clone()); self.recalc_fingerprint(); - self.to_save = Some(ToSave::All) } // This is non-standard. @@ -339,7 +329,6 @@ impl Peerstate { && gossip_header.prefer_encrypt == EncryptPreference::Mutual { self.prefer_encrypt = EncryptPreference::Mutual; - self.to_save = Some(ToSave::All); } }; } @@ -395,7 +384,7 @@ impl Peerstate { if self.public_key_fingerprint.is_some() && self.public_key_fingerprint.as_ref().unwrap() == fingerprint { - self.to_save = Some(ToSave::All); + self.to_save = true; self.verified_key = self.public_key.clone(); self.verified_key_fingerprint = self.public_key_fingerprint.clone(); true @@ -407,7 +396,7 @@ impl Peerstate { if self.gossip_key_fingerprint.is_some() && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint { - self.to_save = Some(ToSave::All); + self.to_save = true; self.verified_key = self.gossip_key.clone(); self.verified_key_fingerprint = self.gossip_key_fingerprint.clone(); true @@ -422,7 +411,7 @@ impl Peerstate { } pub async fn save_to_db(&self, sql: &Sql, create: bool) -> Result<()> { - if self.to_save == Some(ToSave::All) || create { + if self.to_save || create { sql.execute( if create { "INSERT INTO acpeerstates ( \ @@ -467,18 +456,6 @@ impl Peerstate { ], ) .await?; - } else if self.to_save == Some(ToSave::Timestamps) { - sql.execute( - "UPDATE acpeerstates SET last_seen=?, last_seen_autocrypt=?, gossip_timestamp=? \ - WHERE addr=?;", - paramsv![ - self.last_seen, - self.last_seen_autocrypt, - self.gossip_timestamp, - self.addr - ], - ) - .await?; } Ok(()) @@ -651,7 +628,7 @@ pub async fn maybe_do_aeap_transition( "Internal error: Tried to do an AEAP transition without an autocrypt header??", )?; peerstate.apply_header(header, info.message_time); - peerstate.to_save = Some(ToSave::All); + peerstate.to_save = true; // We don't know whether a peerstate with this address already existed, or a // new one should be created, so just try both create=false and create=true, @@ -722,7 +699,7 @@ mod tests { gossip_key_fingerprint: Some(pub_key.fingerprint()), verified_key: Some(pub_key.clone()), verified_key_fingerprint: Some(pub_key.fingerprint()), - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; @@ -737,7 +714,7 @@ mod tests { .expect("no peerstate found in the database"); // clear to_save, as that is not persissted - peerstate.to_save = None; + peerstate.to_save = false; assert_eq!(peerstate, peerstate_new); let peerstate_new2 = Peerstate::from_fingerprint(&ctx.ctx, &pub_key.fingerprint()) .await @@ -764,7 +741,7 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; @@ -797,7 +774,7 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; @@ -811,7 +788,7 @@ mod tests { .expect("failed to load peerstate from db"); // clear to_save, as that is not persissted - peerstate.to_save = None; + peerstate.to_save = false; assert_eq!(Some(peerstate), peerstate_new); } @@ -862,7 +839,7 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: None, + to_save: false, fingerprint_changed: false, }; assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference); diff --git a/src/qr.rs b/src/qr.rs index 800eb4872..2b6a7954a 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -641,7 +641,6 @@ mod tests { use crate::aheader::EncryptPreference; use crate::chat::{create_group_chat, ProtectionStatus}; use crate::key::DcKey; - use crate::peerstate::ToSave; use crate::securejoin::get_securejoin_qr; use crate::test_utils::{alice_keypair, TestContext}; use anyhow::Result; @@ -894,7 +893,7 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; assert!( diff --git a/src/securejoin.rs b/src/securejoin.rs index 81d0bd78d..69ffc0a15 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -18,7 +18,7 @@ use crate::key::{DcKey, Fingerprint, SignedPublicKey}; use crate::message::{Message, Viewtype}; use crate::mimeparser::{MimeMessage, SystemMessage}; use crate::param::Param; -use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus, ToSave}; +use crate::peerstate::{Peerstate, PeerstateKeyType, PeerstateVerifiedStatus}; use crate::qr::check_qr; use crate::stock_str; use crate::token; @@ -640,7 +640,7 @@ async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> PeerstateVerifiedStatus::BidirectVerified, ) { peerstate.prefer_encrypt = EncryptPreference::Mutual; - peerstate.to_save = Some(ToSave::All); + peerstate.to_save = true; peerstate .save_to_db(&context.sql, false) .await @@ -932,7 +932,7 @@ mod tests { gossip_key_fingerprint: Some(alice_pubkey.fingerprint()), verified_key: None, verified_key_fingerprint: None, - to_save: Some(ToSave::All), + to_save: true, fingerprint_changed: false, }; peerstate.save_to_db(&bob.ctx.sql, true).await?; diff --git a/src/tests/aeap.rs b/src/tests/aeap.rs index 7391e87b1..0f52d4a7e 100644 --- a/src/tests/aeap.rs +++ b/src/tests/aeap.rs @@ -341,7 +341,7 @@ async fn mark_as_verified(this: &TestContext, other: &TestContext) { peerstate.verified_key = peerstate.public_key.clone(); peerstate.verified_key_fingerprint = peerstate.public_key_fingerprint.clone(); - peerstate.to_save = Some(peerstate::ToSave::All); + peerstate.to_save = true; peerstate.save_to_db(&this.sql, false).await.unwrap(); } From 62f92d5b28a0527a8876050670c9aff33f1d7931 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 25 Nov 2022 22:08:57 +0000 Subject: [PATCH 11/23] Add UNIQUE constraint to acpeerstates table --- src/sql/migrations.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index a32bd0abd..a02c36ba5 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -616,6 +616,39 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid); ) .await?; } + if dbversion < 94 { + sql.execute_migration( + // Create new `acpeerstates` table, same as before but with unique constraint. + // + // This allows to use `UPSERT` to update existing or insert a new peerstate + // depending on whether one exists already. + "CREATE TABLE new_acpeerstates ( + id INTEGER PRIMARY KEY, + addr TEXT DEFAULT '' COLLATE NOCASE, + last_seen INTEGER DEFAULT 0, + last_seen_autocrypt INTEGER DEFAULT 0, + public_key, + prefer_encrypted INTEGER DEFAULT 0, + gossip_timestamp INTEGER DEFAULT 0, + gossip_key, + public_key_fingerprint TEXT DEFAULT '', + gossip_key_fingerprint TEXT DEFAULT '', + verified_key, + verified_key_fingerprint TEXT DEFAULT '', + UNIQUE (addr) -- Only one peerstate per address + ); + INSERT OR IGNORE INTO new_acpeerstates SELECT * FROM acpeerstates; + DROP TABLE acpeerstates; + ALTER TABLE new_acpeerstates RENAME TO acpeerstates; + CREATE INDEX acpeerstates_index1 ON acpeerstates (addr); + CREATE INDEX acpeerstates_index3 ON acpeerstates (public_key_fingerprint); + CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint); + CREATE INDEX acpeerstates_index5 ON acpeerstates (verified_key_fingerprint); + ", + 94, + ) + .await?; + } let new_version = sql .get_raw_config_int(VERSION_CFG) From c7691fbebe62752bcc10b68875f8f56bb2b3c3bb Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 25 Nov 2022 22:22:29 +0000 Subject: [PATCH 12/23] Use UPSERT when saving peerstates This way there is no need to distinguish between creating and updating peerstate. --- src/decrypt.rs | 4 +-- src/mimeparser.rs | 6 ++-- src/peerstate.rs | 72 +++++++++++++++++++++------------------------- src/qr.rs | 2 +- src/receive_imf.rs | 2 +- src/securejoin.rs | 7 ++--- src/sql.rs | 2 +- src/tests/aeap.rs | 2 +- 8 files changed, 43 insertions(+), 54 deletions(-) diff --git a/src/decrypt.rs b/src/decrypt.rs index 4f76a2786..8c57cdd36 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -306,7 +306,7 @@ pub(crate) async fn get_autocrypt_peerstate( if addr_cmp(&peerstate.addr, from) { if allow_change { peerstate.apply_header(header, message_time); - peerstate.save_to_db(&context.sql, false).await?; + peerstate.save_to_db(&context.sql).await?; } else { info!( context, @@ -322,7 +322,7 @@ pub(crate) async fn get_autocrypt_peerstate( // to the database. } else { let p = Peerstate::from_header(header, message_time); - p.save_to_db(&context.sql, true).await?; + p.save_to_db(&context.sql).await?; peerstate = Some(p); } } else { diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 04a4b4d52..2e811ebb9 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -310,7 +310,7 @@ impl MimeMessage { // && decryption_info.dkim_results.allow_keychange { peerstate.degrade_encryption(message_time); - peerstate.save_to_db(&context.sql, false).await?; + peerstate.save_to_db(&context.sql).await?; } } (Ok(mail), HashSet::new(), false) @@ -1586,11 +1586,11 @@ async fn update_gossip_peerstates( let peerstate; if let Some(mut p) = Peerstate::from_addr(context, &header.addr).await? { p.apply_gossip(&header, message_time); - p.save_to_db(&context.sql, false).await?; + p.save_to_db(&context.sql).await?; peerstate = p; } else { let p = Peerstate::from_gossip(&header, message_time); - p.save_to_db(&context.sql, true).await?; + p.save_to_db(&context.sql).await?; peerstate = p; }; peerstate diff --git a/src/peerstate.rs b/src/peerstate.rs index 09453cac7..11db2c467 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -410,37 +410,34 @@ impl Peerstate { } } - pub async fn save_to_db(&self, sql: &Sql, create: bool) -> Result<()> { - if self.to_save || create { + pub async fn save_to_db(&self, sql: &Sql) -> Result<()> { + if self.to_save { sql.execute( - if create { - "INSERT INTO acpeerstates ( \ - last_seen, \ - last_seen_autocrypt, \ - prefer_encrypted, \ - public_key, \ - gossip_timestamp, \ - gossip_key, \ - public_key_fingerprint, \ - gossip_key_fingerprint, \ - verified_key, \ - verified_key_fingerprint, \ - addr \ - ) VALUES(?,?,?,?,?,?,?,?,?,?,?)" - } else { - "UPDATE acpeerstates \ - SET last_seen=?, \ - last_seen_autocrypt=?, \ - prefer_encrypted=?, \ - public_key=?, \ - gossip_timestamp=?, \ - gossip_key=?, \ - public_key_fingerprint=?, \ - gossip_key_fingerprint=?, \ - verified_key=?, \ - verified_key_fingerprint=? \ - WHERE addr=?" - }, + "INSERT INTO acpeerstates ( + last_seen, + last_seen_autocrypt, + prefer_encrypted, + public_key, + gossip_timestamp, + gossip_key, + public_key_fingerprint, + gossip_key_fingerprint, + verified_key, + verified_key_fingerprint, + addr) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT (addr) + DO UPDATE SET + last_seen = excluded.last_seen, + last_seen_autocrypt = excluded.last_seen_autocrypt, + prefer_encrypted = excluded.prefer_encrypted, + public_key = excluded.public_key, + gossip_timestamp = excluded.gossip_timestamp, + gossip_key = excluded.gossip_key, + public_key_fingerprint = excluded.public_key_fingerprint, + gossip_key_fingerprint = excluded.gossip_key_fingerprint, + verified_key = excluded.verified_key, + verified_key_fingerprint = excluded.verified_key_fingerprint", paramsv![ self.last_seen, self.last_seen_autocrypt, @@ -630,12 +627,7 @@ pub async fn maybe_do_aeap_transition( peerstate.apply_header(header, info.message_time); peerstate.to_save = true; - // We don't know whether a peerstate with this address already existed, or a - // new one should be created, so just try both create=false and create=true, - // and if this fails, create=true, one will succeed (this is a very cold path, - // so performance doesn't really matter). - peerstate.save_to_db(&context.sql, true).await?; - peerstate.save_to_db(&context.sql, false).await?; + peerstate.save_to_db(&context.sql).await?; } } @@ -704,7 +696,7 @@ mod tests { }; assert!( - peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(), + peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(), "failed to save to db" ); @@ -746,11 +738,11 @@ mod tests { }; assert!( - peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(), + peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(), "failed to save" ); assert!( - peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(), + peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(), "double-call with create failed" ); } @@ -779,7 +771,7 @@ mod tests { }; assert!( - peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(), + peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(), "failed to save" ); diff --git a/src/qr.rs b/src/qr.rs index 2b6a7954a..05003e7a9 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -897,7 +897,7 @@ mod tests { fingerprint_changed: false, }; assert!( - peerstate.save_to_db(&ctx.ctx.sql, true).await.is_ok(), + peerstate.save_to_db(&ctx.ctx.sql).await.is_ok(), "failed to save peerstate" ); diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 13241072d..6fbfdddf0 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -2129,7 +2129,7 @@ async fn check_verified_properties( &fp, PeerstateVerifiedStatus::BidirectVerified, ); - peerstate.save_to_db(&context.sql, false).await?; + peerstate.save_to_db(&context.sql).await?; is_verified = true; } } diff --git a/src/securejoin.rs b/src/securejoin.rs index 69ffc0a15..6f977e85a 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -641,10 +641,7 @@ async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> ) { peerstate.prefer_encrypt = EncryptPreference::Mutual; peerstate.to_save = true; - peerstate - .save_to_db(&context.sql, false) - .await - .unwrap_or_default(); + peerstate.save_to_db(&context.sql).await.unwrap_or_default(); return Ok(()); } } @@ -935,7 +932,7 @@ mod tests { to_save: true, fingerprint_changed: false, }; - peerstate.save_to_db(&bob.ctx.sql, true).await?; + peerstate.save_to_db(&bob.ctx.sql).await?; // Step 1: Generate QR-code, ChatId(0) indicates setup-contact let qr = get_securejoin_qr(&alice.ctx, None).await?; diff --git a/src/sql.rs b/src/sql.rs index ce31b5995..3c2d1418d 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -280,7 +280,7 @@ impl Sql { for addr in &addrs { if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await? { peerstate.recalc_fingerprint(); - peerstate.save_to_db(self, false).await?; + peerstate.save_to_db(self).await?; } } } diff --git a/src/tests/aeap.rs b/src/tests/aeap.rs index 0f52d4a7e..90c05f715 100644 --- a/src/tests/aeap.rs +++ b/src/tests/aeap.rs @@ -343,7 +343,7 @@ async fn mark_as_verified(this: &TestContext, other: &TestContext) { peerstate.verified_key_fingerprint = peerstate.public_key_fingerprint.clone(); peerstate.to_save = true; - peerstate.save_to_db(&this.sql, false).await.unwrap(); + peerstate.save_to_db(&this.sql).await.unwrap(); } async fn get_last_info_msg(t: &TestContext, chat_id: ChatId) -> Option { From 98c16ddc4d141bda77c97f720a572a08790ff95e Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 25 Nov 2022 22:55:31 +0000 Subject: [PATCH 13/23] Remove Peerstate.to_save --- src/e2ee.rs | 1 - src/peerstate.rs | 64 ++++++++++++++--------------------------------- src/qr.rs | 1 - src/securejoin.rs | 2 -- src/tests/aeap.rs | 1 - 5 files changed, 19 insertions(+), 50 deletions(-) diff --git a/src/e2ee.rs b/src/e2ee.rs index f8fe11453..311894a04 100644 --- a/src/e2ee.rs +++ b/src/e2ee.rs @@ -296,7 +296,6 @@ Sent with my Delta Chat Messenger: https://delta.chat"; gossip_key_fingerprint: Some(pub_key.fingerprint()), verified_key: Some(pub_key.clone()), verified_key_fingerprint: Some(pub_key.fingerprint()), - to_save: true, fingerprint_changed: false, }; vec![(Some(peerstate), addr)] diff --git a/src/peerstate.rs b/src/peerstate.rs index 11db2c467..34386061f 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -46,7 +46,6 @@ pub struct Peerstate { pub gossip_key_fingerprint: Option, pub verified_key: Option, pub verified_key_fingerprint: Option, - pub to_save: bool, pub fingerprint_changed: bool, } @@ -63,7 +62,6 @@ impl PartialEq for Peerstate { && self.gossip_key_fingerprint == other.gossip_key_fingerprint && self.verified_key == other.verified_key && self.verified_key_fingerprint == other.verified_key_fingerprint - && self.to_save == other.to_save && self.fingerprint_changed == other.fingerprint_changed } } @@ -84,7 +82,6 @@ impl fmt::Debug for Peerstate { .field("gossip_key_fingerprint", &self.gossip_key_fingerprint) .field("verified_key", &self.verified_key) .field("verified_key_fingerprint", &self.verified_key_fingerprint) - .field("to_save", &self.to_save) .field("fingerprint_changed", &self.fingerprint_changed) .finish() } @@ -104,7 +101,6 @@ impl Peerstate { gossip_timestamp: 0, verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, } } @@ -130,7 +126,6 @@ impl Peerstate { gossip_timestamp: message_time, verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, } } @@ -222,7 +217,6 @@ impl Peerstate { .map(|s| s.parse::()) .transpose() .unwrap_or_default(), - to_save: false, fingerprint_changed: false, }; @@ -245,7 +239,6 @@ impl Peerstate { || self.public_key_fingerprint.is_none() || old_public_fingerprint != self.public_key_fingerprint { - self.to_save = true; if old_public_fingerprint.is_some() { self.fingerprint_changed = true; } @@ -260,8 +253,6 @@ impl Peerstate { || self.gossip_key_fingerprint.is_none() || old_gossip_fingerprint != self.gossip_key_fingerprint { - self.to_save = true; - // Warn about gossip key change only if there is no public key obtained from // Autocrypt header, which overrides gossip key. if old_gossip_fingerprint.is_some() && self.public_key_fingerprint.is_none() { @@ -274,7 +265,6 @@ impl Peerstate { pub fn degrade_encryption(&mut self, message_time: i64) { self.prefer_encrypt = EncryptPreference::Reset; self.last_seen = message_time; - self.to_save = true; } pub fn apply_header(&mut self, header: &Aheader, message_time: i64) { @@ -285,7 +275,6 @@ impl Peerstate { if message_time > self.last_seen { self.last_seen = message_time; self.last_seen_autocrypt = message_time; - self.to_save = true; if (header.prefer_encrypt == EncryptPreference::Mutual || header.prefer_encrypt == EncryptPreference::NoPreference) && header.prefer_encrypt != self.prefer_encrypt @@ -307,7 +296,6 @@ impl Peerstate { if message_time > self.gossip_timestamp { self.gossip_timestamp = message_time; - self.to_save = true; if self.gossip_key.as_ref() != Some(&gossip_header.public_key) { self.gossip_key = Some(gossip_header.public_key.clone()); self.recalc_fingerprint(); @@ -384,7 +372,6 @@ impl Peerstate { if self.public_key_fingerprint.is_some() && self.public_key_fingerprint.as_ref().unwrap() == fingerprint { - self.to_save = true; self.verified_key = self.public_key.clone(); self.verified_key_fingerprint = self.public_key_fingerprint.clone(); true @@ -396,7 +383,6 @@ impl Peerstate { if self.gossip_key_fingerprint.is_some() && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint { - self.to_save = true; self.verified_key = self.gossip_key.clone(); self.verified_key_fingerprint = self.gossip_key_fingerprint.clone(); true @@ -411,9 +397,8 @@ impl Peerstate { } pub async fn save_to_db(&self, sql: &Sql) -> Result<()> { - if self.to_save { - sql.execute( - "INSERT INTO acpeerstates ( + sql.execute( + "INSERT INTO acpeerstates ( last_seen, last_seen_autocrypt, prefer_encrypted, @@ -438,23 +423,21 @@ impl Peerstate { gossip_key_fingerprint = excluded.gossip_key_fingerprint, verified_key = excluded.verified_key, verified_key_fingerprint = excluded.verified_key_fingerprint", - paramsv![ - self.last_seen, - self.last_seen_autocrypt, - self.prefer_encrypt as i64, - self.public_key.as_ref().map(|k| k.to_bytes()), - self.gossip_timestamp, - self.gossip_key.as_ref().map(|k| k.to_bytes()), - self.public_key_fingerprint.as_ref().map(|fp| fp.hex()), - self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()), - self.verified_key.as_ref().map(|k| k.to_bytes()), - self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()), - self.addr, - ], - ) - .await?; - } - + paramsv![ + self.last_seen, + self.last_seen_autocrypt, + self.prefer_encrypt as i64, + self.public_key.as_ref().map(|k| k.to_bytes()), + self.gossip_timestamp, + self.gossip_key.as_ref().map(|k| k.to_bytes()), + self.public_key_fingerprint.as_ref().map(|fp| fp.hex()), + self.gossip_key_fingerprint.as_ref().map(|fp| fp.hex()), + self.verified_key.as_ref().map(|k| k.to_bytes()), + self.verified_key_fingerprint.as_ref().map(|fp| fp.hex()), + self.addr, + ], + ) + .await?; Ok(()) } @@ -625,7 +608,6 @@ pub async fn maybe_do_aeap_transition( "Internal error: Tried to do an AEAP transition without an autocrypt header??", )?; peerstate.apply_header(header, info.message_time); - peerstate.to_save = true; peerstate.save_to_db(&context.sql).await?; } @@ -679,7 +661,7 @@ mod tests { let pub_key = alice_keypair().public; - let mut peerstate = Peerstate { + let peerstate = Peerstate { addr: addr.into(), last_seen: 10, last_seen_autocrypt: 11, @@ -691,7 +673,6 @@ mod tests { gossip_key_fingerprint: Some(pub_key.fingerprint()), verified_key: Some(pub_key.clone()), verified_key_fingerprint: Some(pub_key.fingerprint()), - to_save: true, fingerprint_changed: false, }; @@ -705,8 +686,6 @@ mod tests { .expect("failed to load peerstate from db") .expect("no peerstate found in the database"); - // clear to_save, as that is not persissted - peerstate.to_save = false; assert_eq!(peerstate, peerstate_new); let peerstate_new2 = Peerstate::from_fingerprint(&ctx.ctx, &pub_key.fingerprint()) .await @@ -733,7 +712,6 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, }; @@ -754,7 +732,7 @@ mod tests { let pub_key = alice_keypair().public; - let mut peerstate = Peerstate { + let peerstate = Peerstate { addr: addr.into(), last_seen: 10, last_seen_autocrypt: 11, @@ -766,7 +744,6 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, }; @@ -779,8 +756,6 @@ mod tests { .await .expect("failed to load peerstate from db"); - // clear to_save, as that is not persissted - peerstate.to_save = false; assert_eq!(Some(peerstate), peerstate_new); } @@ -831,7 +806,6 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: false, fingerprint_changed: false, }; assert_eq!(peerstate.prefer_encrypt, EncryptPreference::NoPreference); diff --git a/src/qr.rs b/src/qr.rs index 05003e7a9..c12ce8045 100644 --- a/src/qr.rs +++ b/src/qr.rs @@ -893,7 +893,6 @@ mod tests { gossip_key_fingerprint: None, verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, }; assert!( diff --git a/src/securejoin.rs b/src/securejoin.rs index 6f977e85a..08822a5a1 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -640,7 +640,6 @@ async fn mark_peer_as_verified(context: &Context, fingerprint: &Fingerprint) -> PeerstateVerifiedStatus::BidirectVerified, ) { peerstate.prefer_encrypt = EncryptPreference::Mutual; - peerstate.to_save = true; peerstate.save_to_db(&context.sql).await.unwrap_or_default(); return Ok(()); } @@ -929,7 +928,6 @@ mod tests { gossip_key_fingerprint: Some(alice_pubkey.fingerprint()), verified_key: None, verified_key_fingerprint: None, - to_save: true, fingerprint_changed: false, }; peerstate.save_to_db(&bob.ctx.sql).await?; diff --git a/src/tests/aeap.rs b/src/tests/aeap.rs index 90c05f715..fbccd401a 100644 --- a/src/tests/aeap.rs +++ b/src/tests/aeap.rs @@ -341,7 +341,6 @@ async fn mark_as_verified(this: &TestContext, other: &TestContext) { peerstate.verified_key = peerstate.public_key.clone(); peerstate.verified_key_fingerprint = peerstate.public_key_fingerprint.clone(); - peerstate.to_save = true; peerstate.save_to_db(&this.sql).await.unwrap(); } From 264727a414e8f8fdd86d49f5a4f6f8f06fdef6f5 Mon Sep 17 00:00:00 2001 From: link2xt Date: Fri, 25 Nov 2022 22:56:49 +0000 Subject: [PATCH 14/23] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1f003c4c..cd07e0a15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Make sure malformed messsages will never block receiving further messages anymore #3771 - strip leading/trailing whitespace from "Chat-Group-Name{,-Changed}:" headers content #3650 - Assume all Thunderbird users prefer encryption #3774 +- refactor peerstate handling to ensure no duplicate peerstates #3776 ## 1.102.0 From 8855ef72bc0f53e78b9e465837c9eafcf40c8e36 Mon Sep 17 00:00:00 2001 From: link2xt Date: Sat, 26 Nov 2022 00:16:52 +0000 Subject: [PATCH 15/23] Clippy fix --- src/peerstate.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/peerstate.rs b/src/peerstate.rs index 34386061f..d64667306 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -235,13 +235,10 @@ impl Peerstate { let old_public_fingerprint = self.public_key_fingerprint.take(); self.public_key_fingerprint = Some(public_key.fingerprint()); - if old_public_fingerprint.is_none() - || self.public_key_fingerprint.is_none() - || old_public_fingerprint != self.public_key_fingerprint + if old_public_fingerprint.is_some() + && old_public_fingerprint != self.public_key_fingerprint { - if old_public_fingerprint.is_some() { - self.fingerprint_changed = true; - } + self.fingerprint_changed = true; } } From 47743bdcfaeff51deb5c07e26d206e3ca315404c Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 12 Oct 2022 19:41:33 +0000 Subject: [PATCH 16/23] Update `humansize` to version 2 --- Cargo.lock | 7 +++++-- Cargo.toml | 2 +- src/scheduler/connectivity.rs | 10 +++------- src/stock_str.rs | 6 ++---- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8cfc2158d..495355b3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1696,9 +1696,12 @@ dependencies = [ [[package]] name = "humansize" -version = "1.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02296996cb8796d7c6e3bc2d9211b7802812d36999a51bb754123ead7d37d026" +checksum = "4e682e2bd70ecbcce5209f11a992a4ba001fea8e60acf7860ce007629e6d2756" +dependencies = [ + "libm", +] [[package]] name = "humantime" diff --git a/Cargo.toml b/Cargo.toml index aeef4b40f..d24549389 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,7 +74,7 @@ toml = "0.5" url = "2" uuid = { version = "1", features = ["serde", "v4"] } fast-socks5 = "0.8" -humansize = "1" +humansize = "2" qrcodegen = "1.7.0" tagger = "4.3.3" textwrap = "0.16.0" diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index ac2300ad1..a3515daec 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -12,7 +12,7 @@ use crate::tools::time; use crate::{config::Config, scheduler::Scheduler, stock_str, tools}; use crate::{context::Context, log::LogExt}; use anyhow::{anyhow, Result}; -use humansize::{file_size_opts, FileSize}; +use humansize::{format_size, BINARY}; #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)] pub enum Connectivity { @@ -495,12 +495,8 @@ impl Context { // - the string is not longer than the other strings that way (minus title, plus units) - // additional linebreaks on small displays are unlikely therefore // - most times, this is the only item anyway - let usage = (resource.usage * 1024) - .file_size(file_size_opts::BINARY) - .unwrap_or_default(); - let limit = (resource.limit * 1024) - .file_size(file_size_opts::BINARY) - .unwrap_or_default(); + let usage = format_size(resource.usage * 1024, BINARY); + let limit = format_size(resource.limit * 1024, BINARY); stock_str::part_of_total_used(self, usage, limit).await } }; diff --git a/src/stock_str.rs b/src/stock_str.rs index f1b7894db..06b2e952c 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -17,7 +17,7 @@ use crate::context::Context; use crate::message::{Message, Viewtype}; use crate::param::Param; use crate::tools::timestamp_to_str; -use humansize::{file_size_opts, FileSize}; +use humansize::{format_size, BINARY}; #[derive(Debug, Clone)] pub struct StockStrings { @@ -1127,9 +1127,7 @@ pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> St /// Stock string: `%1$s message` with placeholder replaced by human-readable size. pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String { - let size = org_bytes - .file_size(file_size_opts::BINARY) - .unwrap_or_default(); + let size = format_size(org_bytes, BINARY); translated(context, StockMessage::PartialDownloadMsgBody) .await .replace1(size) From e5be023e4bec748be3fe65a92f30457a67511e03 Mon Sep 17 00:00:00 2001 From: missytake Date: Tue, 21 Jun 2022 11:15:41 +0200 Subject: [PATCH 17/23] document provider pull --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 204da5657..71a723eac 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,15 @@ $ cargo test -- --ignored - `vendored`: When using Openssl for TLS, this bundles a vendored version. - `nightly`: Enable nightly only performance and security related features. +## Update Provider Data + +To add the updates from the +[provider-db](https://github.com/deltachat/provider-db) to the core, run: + +``` +./src/provider/update.py ../provider-db/_providers/ > src/provider/data.rs +``` + ## Language bindings and frontend projects Language bindings are available for: From f6a502a8e377636f477382015eb360c1d9cc9e79 Mon Sep 17 00:00:00 2001 From: iequidoo Date: Mon, 28 Nov 2022 14:00:34 -0300 Subject: [PATCH 18/23] Remove the remaining AsRef (#3669) Using &str instead of AsRef is better for compile times, binary size and code complexity. --- CHANGELOG.md | 1 + deltachat-ffi/src/lib.rs | 2 +- deltachat-jsonrpc/src/api/mod.rs | 2 +- src/chat.rs | 9 +- src/config.rs | 20 +++-- src/configure.rs | 4 +- src/ephemeral.rs | 10 +-- src/login_param.rs | 58 ++++++------- src/mimefactory.rs | 4 +- src/peerstate.rs | 2 +- src/scheduler/connectivity.rs | 10 +-- src/sql.rs | 38 ++++---- src/sql/migrations.rs | 6 +- src/stock_str.rs | 144 ++++++++++++++----------------- src/test_utils.rs | 4 +- src/tools.rs | 6 +- 16 files changed, 153 insertions(+), 167 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd07e0a15..7dba5780c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Changes +- Refactor: Remove the remaining AsRef #3669 ### API-Changes - Add Python API to send reactions #3762 diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index 68c7c8d27..f07e4d53b 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -1659,7 +1659,7 @@ pub unsafe extern "C" fn dc_set_chat_profile_image( let ctx = &*context; block_on(async move { - chat::set_chat_profile_image(ctx, ChatId::new(chat_id), to_string_lossy(image)) + chat::set_chat_profile_image(ctx, ChatId::new(chat_id), &to_string_lossy(image)) .await .map(|_| 1) .unwrap_or_log_default(ctx, "Failed to set profile image") diff --git a/deltachat-jsonrpc/src/api/mod.rs b/deltachat-jsonrpc/src/api/mod.rs index e00db7217..a311f8f07 100644 --- a/deltachat-jsonrpc/src/api/mod.rs +++ b/deltachat-jsonrpc/src/api/mod.rs @@ -733,7 +733,7 @@ impl CommandApi { image_path: Option, ) -> Result<()> { let ctx = self.get_context(account_id).await?; - chat::set_chat_profile_image(&ctx, ChatId::new(chat_id), image_path.unwrap_or_default()) + chat::set_chat_profile_image(&ctx, ChatId::new(chat_id), &image_path.unwrap_or_default()) .await } diff --git a/src/chat.rs b/src/chat.rs index 6e58d1d2d..b348f7e09 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2188,7 +2188,7 @@ pub async fn send_videochat_invitation(context: &Context, chat_id: ChatId) -> Re let mut msg = Message::new(Viewtype::VideochatInvitation); msg.param.set(Param::WebrtcRoom, &instance); msg.text = Some( - stock_str::videochat_invite_msg_body(context, Message::parse_webrtc_instance(&instance).1) + stock_str::videochat_invite_msg_body(context, &Message::parse_webrtc_instance(&instance).1) .await, ); send_msg(context, chat_id, &mut msg).await @@ -3005,7 +3005,7 @@ pub async fn set_chat_name(context: &Context, chat_id: ChatId, new_name: &str) - pub async fn set_chat_profile_image( context: &Context, chat_id: ChatId, - new_image: impl AsRef, // XXX use PathBuf + new_image: &str, // XXX use PathBuf ) -> Result<()> { ensure!(!chat_id.is_special(), "Invalid chat ID"); let mut chat = Chat::load_from_db(context, chat_id).await?; @@ -3023,13 +3023,12 @@ pub async fn set_chat_profile_image( let mut msg = Message::new(Viewtype::Text); msg.param .set_int(Param::Cmd, SystemMessage::GroupImageChanged as i32); - if new_image.as_ref().is_empty() { + if new_image.is_empty() { chat.param.remove(Param::ProfileImage); msg.param.remove(Param::Arg); msg.text = Some(stock_str::msg_grp_img_deleted(context, ContactId::SELF).await); } else { - let mut image_blob = - BlobObject::new_from_path(context, Path::new(new_image.as_ref())).await?; + let mut image_blob = BlobObject::new_from_path(context, Path::new(new_image)).await?; image_blob.recode_to_avatar_size(context).await?; chat.param.set(Param::ProfileImage, image_blob.as_name()); msg.param.set(Param::Arg, image_blob.as_name()); diff --git a/src/config.rs b/src/config.rs index e516c0b27..259c3e063 100644 --- a/src/config.rs +++ b/src/config.rs @@ -194,20 +194,20 @@ pub enum Config { impl Context { pub async fn config_exists(&self, key: Config) -> Result { - Ok(self.sql.get_raw_config(key).await?.is_some()) + Ok(self.sql.get_raw_config(key.as_ref()).await?.is_some()) } /// Get a configuration key. Returns `None` if no value is set, and no default value found. pub async fn get_config(&self, key: Config) -> Result> { let value = match key { Config::Selfavatar => { - let rel_path = self.sql.get_raw_config(key).await?; + let rel_path = self.sql.get_raw_config(key.as_ref()).await?; rel_path.map(|p| get_abs_path(self, &p).to_string_lossy().into_owned()) } Config::SysVersion => Some((*DC_VERSION_STR).clone()), Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)), Config::SysConfigKeys => Some(get_config_keys_string()), - _ => self.sql.get_raw_config(key).await?, + _ => self.sql.get_raw_config(key.as_ref()).await?, }; if value.is_some() { @@ -297,26 +297,30 @@ impl Context { Some(value) => { let mut blob = BlobObject::new_from_path(self, value.as_ref()).await?; blob.recode_to_avatar_size(self).await?; - self.sql.set_raw_config(key, Some(blob.as_name())).await?; + self.sql + .set_raw_config(key.as_ref(), Some(blob.as_name())) + .await?; } None => { - self.sql.set_raw_config(key, None).await?; + self.sql.set_raw_config(key.as_ref(), None).await?; } } self.emit_event(EventType::SelfavatarChanged); } Config::DeleteDeviceAfter => { - let ret = self.sql.set_raw_config(key, value).await; + let ret = self.sql.set_raw_config(key.as_ref(), value).await; // Interrupt ephemeral loop to delete old messages immediately. self.interrupt_ephemeral_task().await; ret? } Config::Displayname => { let value = value.map(improve_single_line_input); - self.sql.set_raw_config(key, value.as_deref()).await?; + self.sql + .set_raw_config(key.as_ref(), value.as_deref()) + .await?; } _ => { - self.sql.set_raw_config(key, value).await?; + self.sql.set_raw_config(key.as_ref(), value).await?; } } Ok(()) diff --git a/src/configure.rs b/src/configure.rs index f61c45fed..42d35cdc8 100644 --- a/src/configure.rs +++ b/src/configure.rs @@ -87,7 +87,7 @@ impl Context { self, // We are using Anyhow's .context() and to show the // inner error, too, we need the {:#}: - format!("{:#}", err), + &format!("{:#}", err), ) .await ) @@ -153,7 +153,7 @@ async fn on_configure_completed( if !addr_cmp(&new_addr, &old_addr) { let mut msg = Message::new(Viewtype::Text); msg.text = - Some(stock_str::aeap_explanation_and_link(context, old_addr, new_addr).await); + Some(stock_str::aeap_explanation_and_link(context, &old_addr, &new_addr).await); chat::add_device_msg(context, None, Some(&mut msg)) .await .ok_or_log_msg(context, "Cannot add AEAP explanation"); diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 18e1be385..a65e1636c 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -226,13 +226,13 @@ pub(crate) async fn stock_ephemeral_timer_changed( Timer::Disabled => stock_str::msg_ephemeral_timer_disabled(context, from_id).await, Timer::Enabled { duration } => match duration { 0..=59 => { - stock_str::msg_ephemeral_timer_enabled(context, timer.to_string(), from_id).await + stock_str::msg_ephemeral_timer_enabled(context, &timer.to_string(), from_id).await } 60 => stock_str::msg_ephemeral_timer_minute(context, from_id).await, 61..=3599 => { stock_str::msg_ephemeral_timer_minutes( context, - format!("{}", (f64::from(duration) / 6.0).round() / 10.0), + &format!("{}", (f64::from(duration) / 6.0).round() / 10.0), from_id, ) .await @@ -241,7 +241,7 @@ pub(crate) async fn stock_ephemeral_timer_changed( 3601..=86399 => { stock_str::msg_ephemeral_timer_hours( context, - format!("{}", (f64::from(duration) / 360.0).round() / 10.0), + &format!("{}", (f64::from(duration) / 360.0).round() / 10.0), from_id, ) .await @@ -250,7 +250,7 @@ pub(crate) async fn stock_ephemeral_timer_changed( 86401..=604_799 => { stock_str::msg_ephemeral_timer_days( context, - format!("{}", (f64::from(duration) / 8640.0).round() / 10.0), + &format!("{}", (f64::from(duration) / 8640.0).round() / 10.0), from_id, ) .await @@ -259,7 +259,7 @@ pub(crate) async fn stock_ephemeral_timer_changed( _ => { stock_str::msg_ephemeral_timer_weeks( context, - format!("{}", (f64::from(duration) / 60480.0).round() / 10.0), + &format!("{}", (f64::from(duration) / 60480.0).round() / 10.0), from_id, ) .await diff --git a/src/login_param.rs b/src/login_param.rs index 6d3d724fe..b6749f24f 100644 --- a/src/login_param.rs +++ b/src/login_param.rs @@ -169,7 +169,7 @@ impl LoginParam { async fn from_database(context: &Context, prefix: &str) -> Result { let sql = &context.sql; - let key = format!("{}addr", prefix); + let key = &format!("{}addr", prefix); let addr = sql .get_raw_config(key) .await? @@ -177,26 +177,26 @@ impl LoginParam { .trim() .to_string(); - let key = format!("{}mail_server", prefix); + let key = &format!("{}mail_server", prefix); let mail_server = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}mail_port", prefix); + let key = &format!("{}mail_port", prefix); let mail_port = sql.get_raw_config_int(key).await?.unwrap_or_default(); - let key = format!("{}mail_user", prefix); + let key = &format!("{}mail_user", prefix); let mail_user = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}mail_pw", prefix); + let key = &format!("{}mail_pw", prefix); let mail_pw = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}mail_security", prefix); + let key = &format!("{}mail_security", prefix); let mail_security = sql .get_raw_config_int(key) .await? .and_then(num_traits::FromPrimitive::from_i32) .unwrap_or_default(); - let key = format!("{}imap_certificate_checks", prefix); + let key = &format!("{}imap_certificate_checks", prefix); let imap_certificate_checks = if let Some(certificate_checks) = sql.get_raw_config_int(key).await? { num_traits::FromPrimitive::from_i32(certificate_checks).unwrap() @@ -204,26 +204,26 @@ impl LoginParam { Default::default() }; - let key = format!("{}send_server", prefix); + let key = &format!("{}send_server", prefix); let send_server = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}send_port", prefix); + let key = &format!("{}send_port", prefix); let send_port = sql.get_raw_config_int(key).await?.unwrap_or_default(); - let key = format!("{}send_user", prefix); + let key = &format!("{}send_user", prefix); let send_user = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}send_pw", prefix); + let key = &format!("{}send_pw", prefix); let send_pw = sql.get_raw_config(key).await?.unwrap_or_default(); - let key = format!("{}send_security", prefix); + let key = &format!("{}send_security", prefix); let send_security = sql .get_raw_config_int(key) .await? .and_then(num_traits::FromPrimitive::from_i32) .unwrap_or_default(); - let key = format!("{}smtp_certificate_checks", prefix); + let key = &format!("{}smtp_certificate_checks", prefix); let smtp_certificate_checks = if let Some(certificate_checks) = sql.get_raw_config_int(key).await? { num_traits::FromPrimitive::from_i32(certificate_checks).unwrap_or_default() @@ -231,11 +231,11 @@ impl LoginParam { Default::default() }; - let key = format!("{}server_flags", prefix); + let key = &format!("{}server_flags", prefix); let server_flags = sql.get_raw_config_int(key).await?.unwrap_or_default(); let oauth2 = matches!(server_flags & DC_LP_AUTH_FLAGS, DC_LP_AUTH_OAUTH2); - let key = format!("{}provider", prefix); + let key = &format!("{}provider", prefix); let provider = sql .get_raw_config(key) .await? @@ -275,50 +275,50 @@ impl LoginParam { context.set_primary_self_addr(&self.addr).await?; - let key = format!("{}mail_server", prefix); + let key = &format!("{}mail_server", prefix); sql.set_raw_config(key, Some(&self.imap.server)).await?; - let key = format!("{}mail_port", prefix); + let key = &format!("{}mail_port", prefix); sql.set_raw_config_int(key, i32::from(self.imap.port)) .await?; - let key = format!("{}mail_user", prefix); + let key = &format!("{}mail_user", prefix); sql.set_raw_config(key, Some(&self.imap.user)).await?; - let key = format!("{}mail_pw", prefix); + let key = &format!("{}mail_pw", prefix); sql.set_raw_config(key, Some(&self.imap.password)).await?; - let key = format!("{}mail_security", prefix); + let key = &format!("{}mail_security", prefix); sql.set_raw_config_int(key, self.imap.security as i32) .await?; - let key = format!("{}imap_certificate_checks", prefix); + let key = &format!("{}imap_certificate_checks", prefix); sql.set_raw_config_int(key, self.imap.certificate_checks as i32) .await?; - let key = format!("{}send_server", prefix); + let key = &format!("{}send_server", prefix); sql.set_raw_config(key, Some(&self.smtp.server)).await?; - let key = format!("{}send_port", prefix); + let key = &format!("{}send_port", prefix); sql.set_raw_config_int(key, i32::from(self.smtp.port)) .await?; - let key = format!("{}send_user", prefix); + let key = &format!("{}send_user", prefix); sql.set_raw_config(key, Some(&self.smtp.user)).await?; - let key = format!("{}send_pw", prefix); + let key = &format!("{}send_pw", prefix); sql.set_raw_config(key, Some(&self.smtp.password)).await?; - let key = format!("{}send_security", prefix); + let key = &format!("{}send_security", prefix); sql.set_raw_config_int(key, self.smtp.security as i32) .await?; - let key = format!("{}smtp_certificate_checks", prefix); + let key = &format!("{}smtp_certificate_checks", prefix); sql.set_raw_config_int(key, self.smtp.certificate_checks as i32) .await?; // The OAuth2 flag is either set for both IMAP and SMTP or not at all. - let key = format!("{}server_flags", prefix); + let key = &format!("{}server_flags", prefix); let server_flags = match self.imap.oauth2 { true => DC_LP_AUTH_OAUTH2, false => DC_LP_AUTH_NORMAL, @@ -326,7 +326,7 @@ impl LoginParam { sql.set_raw_config_int(key, server_flags).await?; if let Some(provider) = self.provider { - let key = format!("{}provider", prefix); + let key = &format!("{}provider", prefix); sql.set_raw_config(key, Some(provider.id)).await?; } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index b6f0fef04..4a1f9a6e1 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -429,7 +429,7 @@ impl<'a> MimeFactory<'a> { } } - let self_name = match context.get_config(Config::Displayname).await? { + let self_name = &match context.get_config(Config::Displayname).await? { Some(name) => name, None => context.get_config(Config::Addr).await?.unwrap_or_default(), }; @@ -1259,7 +1259,7 @@ impl<'a> MimeFactory<'a> { .truncated_text(32) .to_string() }; - let p2 = stock_str::read_rcpt_mail_body(context, p1).await; + let p2 = stock_str::read_rcpt_mail_body(context, &p1).await; let message_text = format!("{}\r\n", format_flowed(&p2)); message = message.child( PartBuilder::new() diff --git a/src/peerstate.rs b/src/peerstate.rs index d64667306..dd70f2b8a 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -474,7 +474,7 @@ impl Peerstate { let chats = Chatlist::try_load(context, 0, None, Some(contact_id)).await?; let msg = match &change { PeerstateChange::FingerprintChange => { - stock_str::contact_setup_changed(context, self.addr.clone()).await + stock_str::contact_setup_changed(context, &self.addr).await } PeerstateChange::Aeap(new_addr) => { let old_contact = Contact::load_from_db(context, contact_id).await?; diff --git a/src/scheduler/connectivity.rs b/src/scheduler/connectivity.rs index a3515daec..9ca9fb263 100644 --- a/src/scheduler/connectivity.rs +++ b/src/scheduler/connectivity.rs @@ -448,7 +448,7 @@ impl Context { // [======67%===== ] // ============================================================================================= - let domain = tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain; + let domain = &tools::EmailAddress::new(&self.get_primary_self_addr().await?)?.domain; let storage_on_domain = stock_str::storage_on_domain(self, domain).await; ret += &format!("

{}

    ", storage_on_domain); let quota = self.quota.read().await; @@ -473,8 +473,8 @@ impl Context { let messages = stock_str::messages(self).await; let part_of_total_used = stock_str::part_of_total_used( self, - resource.usage.to_string(), - resource.limit.to_string(), + &resource.usage.to_string(), + &resource.limit.to_string(), ) .await; ret += &match &resource.name { @@ -495,8 +495,8 @@ impl Context { // - the string is not longer than the other strings that way (minus title, plus units) - // additional linebreaks on small displays are unlikely therefore // - most times, this is the only item anyway - let usage = format_size(resource.usage * 1024, BINARY); - let limit = format_size(resource.limit * 1024, BINARY); + let usage = &format_size(resource.usage * 1024, BINARY); + let limit = &format_size(resource.limit * 1024, BINARY); stock_str::part_of_total_used(self, usage, limit).await } }; diff --git a/src/sql.rs b/src/sql.rs index 3c2d1418d..d1e1c7e2d 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -528,9 +528,7 @@ impl Sql { /// /// Setting `None` deletes the value. On failure an error message /// will already have been logged. - pub async fn set_raw_config(&self, key: impl AsRef, value: Option<&str>) -> Result<()> { - let key = key.as_ref(); - + pub async fn set_raw_config(&self, key: &str, value: Option<&str>) -> Result<()> { let mut lock = self.config_cache.write().await; if let Some(value) = value { let exists = self @@ -564,9 +562,9 @@ impl Sql { } /// Get configuration options from the database. - pub async fn get_raw_config(&self, key: impl AsRef) -> Result> { + pub async fn get_raw_config(&self, key: &str) -> Result> { let lock = self.config_cache.read().await; - let cached = lock.get(key.as_ref()).cloned(); + let cached = lock.get(key).cloned(); drop(lock); if let Some(c) = cached { @@ -575,48 +573,42 @@ impl Sql { let mut lock = self.config_cache.write().await; let value = self - .query_get_value( - "SELECT value FROM config WHERE keyname=?;", - paramsv![key.as_ref()], - ) + .query_get_value("SELECT value FROM config WHERE keyname=?;", paramsv![key]) .await - .context(format!("failed to fetch raw config: {}", key.as_ref()))?; - lock.insert(key.as_ref().to_string(), value.clone()); + .context(format!("failed to fetch raw config: {}", key))?; + lock.insert(key.to_string(), value.clone()); drop(lock); Ok(value) } - pub async fn set_raw_config_int(&self, key: impl AsRef, value: i32) -> Result<()> { + pub async fn set_raw_config_int(&self, key: &str, value: i32) -> Result<()> { self.set_raw_config(key, Some(&format!("{}", value))).await } - pub async fn get_raw_config_int(&self, key: impl AsRef) -> Result> { + pub async fn get_raw_config_int(&self, key: &str) -> Result> { self.get_raw_config(key) .await .map(|s| s.and_then(|s| s.parse().ok())) } - pub async fn get_raw_config_bool(&self, key: impl AsRef) -> Result { + pub async fn get_raw_config_bool(&self, key: &str) -> Result { // Not the most obvious way to encode bool as string, but it is matter // of backward compatibility. let res = self.get_raw_config_int(key).await?; Ok(res.unwrap_or_default() > 0) } - pub async fn set_raw_config_bool(&self, key: T, value: bool) -> Result<()> - where - T: AsRef, - { + pub async fn set_raw_config_bool(&self, key: &str, value: bool) -> Result<()> { let value = if value { Some("1") } else { None }; self.set_raw_config(key, value).await } - pub async fn set_raw_config_int64(&self, key: impl AsRef, value: i64) -> Result<()> { + pub async fn set_raw_config_int64(&self, key: &str, value: i64) -> Result<()> { self.set_raw_config(key, Some(&format!("{}", value))).await } - pub async fn get_raw_config_int64(&self, key: impl AsRef) -> Result> { + pub async fn get_raw_config_int64(&self, key: &str) -> Result> { self.get_raw_config(key) .await .map(|s| s.and_then(|r| r.parse().ok())) @@ -728,7 +720,7 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> { |row| row.get::<_, String>(0), |rows| { for row in rows { - maybe_add_file(&mut files_in_use, row?); + maybe_add_file(&mut files_in_use, &row?); } Ok(()) }, @@ -819,8 +811,8 @@ fn is_file_in_use(files_in_use: &HashSet, namespc_opt: Option<&str>, nam files_in_use.contains(name_to_check) } -fn maybe_add_file(files_in_use: &mut HashSet, file: impl AsRef) { - if let Some(file) = file.as_ref().strip_prefix("$BLOBDIR/") { +fn maybe_add_file(files_in_use: &mut HashSet, file: &str) { + if let Some(file) = file.strip_prefix("$BLOBDIR/") { files_in_use.insert(file.to_string()); } } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index a02c36ba5..fcb69cbac 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -320,11 +320,11 @@ ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0;"#, if dbversion < 67 { for prefix in &["", "configured_"] { if let Some(server_flags) = sql - .get_raw_config_int(format!("{}server_flags", prefix)) + .get_raw_config_int(&format!("{}server_flags", prefix)) .await? { let imap_socket_flags = server_flags & 0x700; - let key = format!("{}mail_security", prefix); + let key = &format!("{}mail_security", prefix); match imap_socket_flags { 0x100 => sql.set_raw_config_int(key, 2).await?, // STARTTLS 0x200 => sql.set_raw_config_int(key, 1).await?, // SSL/TLS @@ -332,7 +332,7 @@ ALTER TABLE msgs ADD COLUMN ephemeral_timestamp INTEGER DEFAULT 0;"#, _ => sql.set_raw_config_int(key, 0).await?, } let smtp_socket_flags = server_flags & 0x70000; - let key = format!("{}send_security", prefix); + let key = &format!("{}send_security", prefix); match smtp_socket_flags { 0x10000 => sql.set_raw_config_int(key, 2).await?, // STARTTLS 0x20000 => sql.set_raw_config_int(key, 1).await?, // SSL/TLS diff --git a/src/stock_str.rs b/src/stock_str.rs index 06b2e952c..e58de954d 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -466,33 +466,33 @@ async fn translated(context: &Context, id: StockMessage) -> String { /// Helper trait only meant to be implemented for [`String`]. trait StockStringMods: AsRef + Sized { /// Substitutes the first replacement value if one is present. - fn replace1(&self, replacement: impl AsRef) -> String { + fn replace1(&self, replacement: &str) -> String { self.as_ref() - .replacen("%1$s", replacement.as_ref(), 1) - .replacen("%1$d", replacement.as_ref(), 1) - .replacen("%1$@", replacement.as_ref(), 1) + .replacen("%1$s", replacement, 1) + .replacen("%1$d", replacement, 1) + .replacen("%1$@", replacement, 1) } /// Substitutes the second replacement value if one is present. /// /// Be aware you probably should have also called [`StockStringMods::replace1`] if /// you are calling this. - fn replace2(&self, replacement: impl AsRef) -> String { + fn replace2(&self, replacement: &str) -> String { self.as_ref() - .replacen("%2$s", replacement.as_ref(), 1) - .replacen("%2$d", replacement.as_ref(), 1) - .replacen("%2$@", replacement.as_ref(), 1) + .replacen("%2$s", replacement, 1) + .replacen("%2$d", replacement, 1) + .replacen("%2$@", replacement, 1) } /// Substitutes the third replacement value if one is present. /// /// Be aware you probably should have also called [`StockStringMods::replace1`] and /// [`StockStringMods::replace2`] if you are calling this. - fn replace3(&self, replacement: impl AsRef) -> String { + fn replace3(&self, replacement: &str) -> String { self.as_ref() - .replacen("%3$s", replacement.as_ref(), 1) - .replacen("%3$d", replacement.as_ref(), 1) - .replacen("%3$@", replacement.as_ref(), 1) + .replacen("%3$s", replacement, 1) + .replacen("%3$d", replacement, 1) + .replacen("%3$@", replacement, 1) } } @@ -551,8 +551,8 @@ pub(crate) async fn file(context: &Context) -> String { /// Stock string: `Group name changed from "%1$s" to "%2$s".`. pub(crate) async fn msg_grp_name( context: &Context, - from_group: impl AsRef, - to_group: impl AsRef, + from_group: &str, + to_group: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -565,7 +565,7 @@ pub(crate) async fn msg_grp_name( .await .replace1(from_group) .replace2(to_group) - .replace3(by_contact.get_stock_name(context).await) + .replace3(&by_contact.get_stock_name(context).await) } } @@ -575,7 +575,7 @@ pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: ContactId } else { translated(context, StockMessage::MsgGrpImgChangedBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -585,11 +585,11 @@ pub(crate) async fn msg_grp_img_changed(context: &Context, by_contact: ContactId /// contacts to combine with the display name. pub(crate) async fn msg_add_member( context: &Context, - added_member_addr: impl AsRef, + added_member_addr: &str, by_contact: ContactId, ) -> String { - let addr = added_member_addr.as_ref(); - let who = match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await { + let addr = added_member_addr; + let who = &match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await { Ok(Some(contact_id)) => Contact::get_by_id(context, contact_id) .await .map(|contact| contact.get_name_n_addr()) @@ -604,7 +604,7 @@ pub(crate) async fn msg_add_member( translated(context, StockMessage::MsgAddMemberBy) .await .replace1(who) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } @@ -614,11 +614,11 @@ pub(crate) async fn msg_add_member( /// the contacts to combine with the display name. pub(crate) async fn msg_del_member( context: &Context, - removed_member_addr: impl AsRef, + removed_member_addr: &str, by_contact: ContactId, ) -> String { - let addr = removed_member_addr.as_ref(); - let who = match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await { + let addr = removed_member_addr; + let who = &match Contact::lookup_id_by_addr(context, addr, Origin::Unknown).await { Ok(Some(contact_id)) => Contact::get_by_id(context, contact_id) .await .map(|contact| contact.get_name_n_addr()) @@ -633,7 +633,7 @@ pub(crate) async fn msg_del_member( translated(context, StockMessage::MsgDelMemberBy) .await .replace1(who) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } @@ -644,7 +644,7 @@ pub(crate) async fn msg_group_left(context: &Context, by_contact: ContactId) -> } else { translated(context, StockMessage::MsgGroupLeftBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -684,7 +684,7 @@ pub(crate) async fn read_rcpt(context: &Context) -> String { } /// Stock string: `This is a return receipt for the message "%1$s".`. -pub(crate) async fn read_rcpt_mail_body(context: &Context, message: impl AsRef) -> String { +pub(crate) async fn read_rcpt_mail_body(context: &Context, message: &str) -> String { translated(context, StockMessage::ReadRcptMailBody) .await .replace1(message) @@ -697,7 +697,7 @@ pub(crate) async fn msg_grp_img_deleted(context: &Context, by_contact: ContactId } else { translated(context, StockMessage::MsgGrpImgDeletedBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -714,7 +714,7 @@ pub(crate) async fn secure_join_started( if let Ok(contact) = Contact::get_by_id(context, inviter_contact_id).await { translated(context, StockMessage::SecureJoinStarted) .await - .replace1(contact.get_name_n_addr()) + .replace1(&contact.get_name_n_addr()) .replace2(contact.get_display_name()) } else { format!( @@ -741,7 +741,7 @@ pub(crate) async fn setup_contact_qr_description( display_name: &str, addr: &str, ) -> String { - let name = if display_name == addr { + let name = &if display_name == addr { addr.to_owned() } else { format!("{} ({})", display_name, addr) @@ -760,7 +760,7 @@ pub(crate) async fn secure_join_group_qr_description(context: &Context, chat: &C /// Stock string: `%1$s verified.`. pub(crate) async fn contact_verified(context: &Context, contact: &Contact) -> String { - let addr = contact.get_name_n_addr(); + let addr = &contact.get_name_n_addr(); translated(context, StockMessage::ContactVerified) .await .replace1(addr) @@ -768,17 +768,14 @@ pub(crate) async fn contact_verified(context: &Context, contact: &Contact) -> St /// Stock string: `Cannot verify %1$s`. pub(crate) async fn contact_not_verified(context: &Context, contact: &Contact) -> String { - let addr = contact.get_name_n_addr(); + let addr = &contact.get_name_n_addr(); translated(context, StockMessage::ContactNotVerified) .await .replace1(addr) } /// Stock string: `Changed setup for %1$s`. -pub(crate) async fn contact_setup_changed( - context: &Context, - contact_addr: impl AsRef, -) -> String { +pub(crate) async fn contact_setup_changed(context: &Context, contact_addr: &str) -> String { translated(context, StockMessage::ContactSetupChanged) .await .replace1(contact_addr) @@ -810,7 +807,7 @@ pub(crate) async fn sync_msg_body(context: &Context) -> String { } /// Stock string: `Cannot login as \"%1$s\". Please check...`. -pub(crate) async fn cannot_login(context: &Context, user: impl AsRef) -> String { +pub(crate) async fn cannot_login(context: &Context, user: &str) -> String { translated(context, StockMessage::CannotLogin) .await .replace1(user) @@ -828,7 +825,7 @@ pub(crate) async fn msg_location_enabled_by(context: &Context, contact: ContactI } else { translated(context, StockMessage::MsgLocationEnabledBy) .await - .replace1(contact.get_stock_name(context).await) + .replace1(&contact.get_stock_name(context).await) } } @@ -874,17 +871,14 @@ pub(crate) async fn unknown_sender_for_chat(context: &Context) -> String { /// Stock string: `Message from %1$s`. // TODO: This can compute `self_name` itself instead of asking the caller to do this. -pub(crate) async fn subject_for_new_contact( - context: &Context, - self_name: impl AsRef, -) -> String { +pub(crate) async fn subject_for_new_contact(context: &Context, self_name: &str) -> String { translated(context, StockMessage::SubjectForNewContact) .await .replace1(self_name) } /// Stock string: `Failed to send message to %1$s.`. -pub(crate) async fn failed_sending_to(context: &Context, name: impl AsRef) -> String { +pub(crate) async fn failed_sending_to(context: &Context, name: &str) -> String { translated(context, StockMessage::FailedSendingTo) .await .replace1(name) @@ -900,14 +894,14 @@ pub(crate) async fn msg_ephemeral_timer_disabled( } else { translated(context, StockMessage::MsgEphemeralTimerDisabledBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } /// Stock string: `Message deletion timer is set to %1$s s.`. pub(crate) async fn msg_ephemeral_timer_enabled( context: &Context, - timer: impl AsRef, + timer: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -918,7 +912,7 @@ pub(crate) async fn msg_ephemeral_timer_enabled( translated(context, StockMessage::MsgEphemeralTimerEnabledBy) .await .replace1(timer) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } @@ -929,7 +923,7 @@ pub(crate) async fn msg_ephemeral_timer_minute(context: &Context, by_contact: Co } else { translated(context, StockMessage::MsgEphemeralTimerMinuteBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -940,7 +934,7 @@ pub(crate) async fn msg_ephemeral_timer_hour(context: &Context, by_contact: Cont } else { translated(context, StockMessage::MsgEphemeralTimerHourBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -951,7 +945,7 @@ pub(crate) async fn msg_ephemeral_timer_day(context: &Context, by_contact: Conta } else { translated(context, StockMessage::MsgEphemeralTimerDayBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -962,7 +956,7 @@ pub(crate) async fn msg_ephemeral_timer_week(context: &Context, by_contact: Cont } else { translated(context, StockMessage::MsgEphemeralTimerWeekBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -972,14 +966,14 @@ pub(crate) async fn videochat_invitation(context: &Context) -> String { } /// Stock string: `You are invited to a video chat, click %1$s to join.`. -pub(crate) async fn videochat_invite_msg_body(context: &Context, url: impl AsRef) -> String { +pub(crate) async fn videochat_invite_msg_body(context: &Context, url: &str) -> String { translated(context, StockMessage::VideochatInviteMsgBody) .await .replace1(url) } /// Stock string: `Error:\n\n“%1$s”`. -pub(crate) async fn configuration_failed(context: &Context, details: impl AsRef) -> String { +pub(crate) async fn configuration_failed(context: &Context, details: &str) -> String { translated(context, StockMessage::ConfigurationFailed) .await .replace1(details) @@ -987,7 +981,7 @@ pub(crate) async fn configuration_failed(context: &Context, details: impl AsRef< /// Stock string: `⚠️ Date or time of your device seem to be inaccurate (%1$s)...`. // TODO: This could compute now itself. -pub(crate) async fn bad_time_msg_body(context: &Context, now: impl AsRef) -> String { +pub(crate) async fn bad_time_msg_body(context: &Context, now: &str) -> String { translated(context, StockMessage::BadTimeMsgBody) .await .replace1(now) @@ -1010,7 +1004,7 @@ pub(crate) async fn protection_enabled(context: &Context, by_contact: ContactId) } else { translated(context, StockMessage::ProtectionEnabledBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -1021,7 +1015,7 @@ pub(crate) async fn protection_disabled(context: &Context, by_contact: ContactId } else { translated(context, StockMessage::ProtectionDisabledBy) .await - .replace1(by_contact.get_stock_name(context).await) + .replace1(&by_contact.get_stock_name(context).await) } } @@ -1043,7 +1037,7 @@ pub(crate) async fn delete_server_turned_off(context: &Context) -> String { /// Stock string: `Message deletion timer is set to %1$s minutes.`. pub(crate) async fn msg_ephemeral_timer_minutes( context: &Context, - minutes: impl AsRef, + minutes: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -1054,14 +1048,14 @@ pub(crate) async fn msg_ephemeral_timer_minutes( translated(context, StockMessage::MsgEphemeralTimerMinutesBy) .await .replace1(minutes) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } /// Stock string: `Message deletion timer is set to %1$s hours.`. pub(crate) async fn msg_ephemeral_timer_hours( context: &Context, - hours: impl AsRef, + hours: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -1072,14 +1066,14 @@ pub(crate) async fn msg_ephemeral_timer_hours( translated(context, StockMessage::MsgEphemeralTimerHoursBy) .await .replace1(hours) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } /// Stock string: `Message deletion timer is set to %1$s days.`. pub(crate) async fn msg_ephemeral_timer_days( context: &Context, - days: impl AsRef, + days: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -1090,14 +1084,14 @@ pub(crate) async fn msg_ephemeral_timer_days( translated(context, StockMessage::MsgEphemeralTimerDaysBy) .await .replace1(days) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } /// Stock string: `Message deletion timer is set to %1$s weeks.`. pub(crate) async fn msg_ephemeral_timer_weeks( context: &Context, - weeks: impl AsRef, + weeks: &str, by_contact: ContactId, ) -> String { if by_contact == ContactId::SELF { @@ -1108,7 +1102,7 @@ pub(crate) async fn msg_ephemeral_timer_weeks( translated(context, StockMessage::MsgEphemeralTimerWeeksBy) .await .replace1(weeks) - .replace2(by_contact.get_stock_name(context).await) + .replace2(&by_contact.get_stock_name(context).await) } } @@ -1121,13 +1115,13 @@ pub(crate) async fn forwarded(context: &Context) -> String { pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> String { translated(context, StockMessage::QuotaExceedingMsgBody) .await - .replace1(format!("{}", highest_usage)) + .replace1(&format!("{}", highest_usage)) .replace("%%", "%") } /// Stock string: `%1$s message` with placeholder replaced by human-readable size. pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String { - let size = format_size(org_bytes, BINARY); + let size = &format_size(org_bytes, BINARY); translated(context, StockMessage::PartialDownloadMsgBody) .await .replace1(size) @@ -1137,7 +1131,7 @@ pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String { translated(context, StockMessage::DownloadAvailability) .await - .replace1(timestamp_to_str(timestamp)) + .replace1(×tamp_to_str(timestamp)) } /// Stock string: `Incoming Messages`. @@ -1152,7 +1146,7 @@ pub(crate) async fn outgoing_messages(context: &Context) -> String { /// Stock string: `Storage on %1$s`. /// `%1$s` will be replaced by the domain of the configured email-address. -pub(crate) async fn storage_on_domain(context: &Context, domain: impl AsRef) -> String { +pub(crate) async fn storage_on_domain(context: &Context, domain: &str) -> String { translated(context, StockMessage::StorageOnDomain) .await .replace1(domain) @@ -1190,7 +1184,7 @@ pub(crate) async fn last_msg_sent_successfully(context: &Context) -> String { /// Stock string: `Error: %1$s…`. /// `%1$s` will be replaced by a possibly more detailed, typically english, error description. -pub(crate) async fn error(context: &Context, error: impl AsRef) -> String { +pub(crate) async fn error(context: &Context, error: &str) -> String { translated(context, StockMessage::Error) .await .replace1(error) @@ -1208,11 +1202,7 @@ pub(crate) async fn messages(context: &Context) -> String { } /// Stock string: `%1$s of %2$s used`. -pub(crate) async fn part_of_total_used( - context: &Context, - part: impl AsRef, - total: impl AsRef, -) -> String { +pub(crate) async fn part_of_total_used(context: &Context, part: &str, total: &str) -> String { translated(context, StockMessage::PartOfTotallUsed) .await .replace1(part) @@ -1228,9 +1218,9 @@ pub(crate) async fn broadcast_list(context: &Context) -> String { /// Stock string: `%1$s changed their address from %2$s to %3$s`. pub(crate) async fn aeap_addr_changed( context: &Context, - contact_name: impl AsRef, - old_addr: impl AsRef, - new_addr: impl AsRef, + contact_name: &str, + old_addr: &str, + new_addr: &str, ) -> String { translated(context, StockMessage::AeapAddrChanged) .await @@ -1241,8 +1231,8 @@ pub(crate) async fn aeap_addr_changed( pub(crate) async fn aeap_explanation_and_link( context: &Context, - old_addr: impl AsRef, - new_addr: impl AsRef, + old_addr: &str, + new_addr: &str, ) -> String { translated(context, StockMessage::AeapExplanationAndLink) .await diff --git a/src/test_utils.rs b/src/test_utils.rs index 243edf325..472087563 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -979,7 +979,7 @@ fn print_event(event: &Event) { /// Logs an individual message to stdout. /// /// This includes a bunch of the message meta-data as well. -async fn log_msg(context: &Context, prefix: impl AsRef, msg: &Message) { +async fn log_msg(context: &Context, prefix: &str, msg: &Message) { let contact = match Contact::get_by_id(context, msg.get_from_id()).await { Ok(contact) => contact, Err(e) => { @@ -1001,7 +1001,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef, msg: &Message) { let msgtext = msg.get_text(); println!( "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}", - prefix.as_ref(), + prefix, msg.get_id(), if msg.get_showpadlock() { "🔒" } else { "" }, if msg.has_location() { "📍" } else { "" }, diff --git a/src/tools.rs b/src/tools.rs index a9c0a47ea..8d68e32b1 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -206,7 +206,7 @@ async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestam msg.text = Some( stock_str::bad_time_msg_body( context, - Local + &Local .timestamp(now, 0) .format("%Y-%m-%d %H:%M:%S") .to_string(), @@ -319,8 +319,8 @@ pub(crate) fn extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> { } // the returned suffix is lower-case -pub fn get_filesuffix_lc(path_filename: impl AsRef) -> Option { - Path::new(path_filename.as_ref()) +pub fn get_filesuffix_lc(path_filename: &str) -> Option { + Path::new(path_filename) .extension() .map(|p| p.to_string_lossy().to_lowercase()) } From 6c4d9198287b39b21e130a04d1e4c7c91efc564c Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 29 Nov 2022 20:37:56 +0100 Subject: [PATCH 19/23] Add members to chats in a single sql transation (#3780) This esp. speeds up receive_imf a bit when we recreate the member list (recreate_member_list == true). It's a preparation for https://github.com/deltachat/deltachat-core-rust/issues/3768, which will be a one-line-fix, but recreate the member list more often, so that we want to optimize this case a bit. It also adds a benchmark for this case. It's not that easy to make the benchmark non-flaky, but by closing all other programs and locking the CPU to 1.5GHz it worked. It is consistently a few percent faster than ./without-optim: ``` Receive messages/Receive 100 Chat-Group-Member-{Added|Removed} messages time: [52.257 ms 52.569 ms 52.941 ms] change: [-3.5301% -2.6181% -1.6697%] (p = 0.00 < 0.05) Performance has improved. Found 7 outliers among 100 measurements (7.00%) 4 (4.00%) high mild 3 (3.00%) high severe ``` --- CHANGELOG.md | 1 + benches/receive_emails.rs | 71 +++++++++++++++++++++++++++++++++++++-- src/chat.rs | 22 +++++++----- src/peerstate.rs | 6 ++-- src/receive_imf.rs | 69 +++++++++++++++---------------------- src/securejoin/bob.rs | 2 +- src/sql.rs | 2 +- src/sql/migrations.rs | 11 ++++++ 8 files changed, 127 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dba5780c..fea7c2fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changes - Refactor: Remove the remaining AsRef #3669 +- Small speedup #3780 ### API-Changes - Add Python API to send reactions #3762 diff --git a/benches/receive_emails.rs b/benches/receive_emails.rs index 1e9e5aa43..ece40985e 100644 --- a/benches/receive_emails.rs +++ b/benches/receive_emails.rs @@ -38,11 +38,64 @@ Hello {i}", context } +/// Receive 100 emails that remove charlie@example.com and add +/// him back +async fn recv_groupmembership_emails(context: Context) -> Context { + for i in 0..50 { + let imf_raw = format!( + "Subject: Benchmark +Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org +Date: Sat, 07 Dec 2019 19:00:27 +0000 +To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com +From: sender@testrun.org +Chat-Version: 1.0 +Chat-Disposition-Notification-To: sender@testrun.org +Chat-User-Avatar: 0 +Chat-Group-Member-Added: charlie@example.com +In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org +MIME-Version: 1.0 + +Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no + +Hello {i}", + i = i, + i_dec = i - 1, + ); + receive_imf(&context, black_box(imf_raw.as_bytes()), false) + .await + .unwrap(); + + let imf_raw = format!( + "Subject: Benchmark +Message-ID: Gr.OssSYnOFkhR.{i}@testrun.org +Date: Sat, 07 Dec 2019 19:00:27 +0000 +To: alice@example.com, b@example.com, c@example.com, d@example.com, e@example.com, f@example.com +From: sender@testrun.org +Chat-Version: 1.0 +Chat-Disposition-Notification-To: sender@testrun.org +Chat-User-Avatar: 0 +Chat-Group-Member-Removed: charlie@example.com +In-Reply-To: Gr.OssSYnOFkhR.{i_dec}@testrun.org +MIME-Version: 1.0 + +Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no + +Hello {i}", + i = i, + i_dec = i - 1, + ); + receive_imf(&context, black_box(imf_raw.as_bytes()), false) + .await + .unwrap(); + } + context +} + async fn create_context() -> Context { let dir = tempdir().unwrap(); let dbfile = dir.path().join("db.sqlite"); let id = 100; - let context = Context::new(&dbfile, id, Events::new(), StockStrings::new()) + let context = Context::new(dbfile.as_path(), id, Events::new(), StockStrings::new()) .await .unwrap(); @@ -52,7 +105,7 @@ async fn create_context() -> Context { if backup.exists() { println!("Importing backup"); - imex(&context, ImexMode::ImportBackup, &backup, None) + imex(&context, ImexMode::ImportBackup, backup.as_path(), None) .await .unwrap(); } @@ -83,6 +136,20 @@ fn criterion_benchmark(c: &mut Criterion) { } }); }); + group.bench_function( + "Receive 100 Chat-Group-Member-{Added|Removed} messages", + |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + let context = rt.block_on(create_context()); + + b.to_async(&rt).iter(|| { + let ctx = context.clone(); + async move { + recv_groupmembership_emails(black_box(ctx)).await; + } + }); + }, + ); group.finish(); } diff --git a/src/chat.rs b/src/chat.rs index b348f7e09..9d1693924 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2563,7 +2563,7 @@ pub async fn create_group_chat( let chat_id = ChatId::new(u32::try_from(row_id)?); if !is_contact_in_chat(context, chat_id, ContactId::SELF).await? { - add_to_chat_contacts_table(context, chat_id, ContactId::SELF).await?; + add_to_chat_contacts_table(context, chat_id, &[ContactId::SELF]).await?; } context.emit_msgs_changed_without_ids(); @@ -2624,19 +2624,25 @@ pub async fn create_broadcast_list(context: &Context) -> Result { Ok(chat_id) } -/// Adds a contact to the `chats_contacts` table. +/// Adds contacts to the `chats_contacts` table. pub(crate) async fn add_to_chat_contacts_table( context: &Context, chat_id: ChatId, - contact_id: ContactId, + contact_ids: &[ContactId], ) -> Result<()> { context .sql - .execute( - "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)", - paramsv![chat_id, contact_id], - ) + .transaction(move |transaction| { + for contact_id in contact_ids { + transaction.execute( + "INSERT OR IGNORE INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)", + paramsv![chat_id, contact_id], + )?; + } + Ok(()) + }) .await?; + Ok(()) } @@ -2738,7 +2744,7 @@ pub(crate) async fn add_contact_to_chat_ex( if is_contact_in_chat(context, chat_id, contact_id).await? { return Ok(false); } - add_to_chat_contacts_table(context, chat_id, contact_id).await?; + add_to_chat_contacts_table(context, chat_id, &[contact_id]).await?; } if chat.typ == Chattype::Group && chat.is_promoted() { msg.viewtype = Viewtype::Text; diff --git a/src/peerstate.rs b/src/peerstate.rs index dd70f2b8a..e9591c789 100644 --- a/src/peerstate.rs +++ b/src/peerstate.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::fmt; use crate::aheader::{Aheader, EncryptPreference}; -use crate::chat::{self, is_contact_in_chat, Chat}; +use crate::chat::{self, Chat}; use crate::chatlist::Chatlist; use crate::constants::Chattype; use crate::contact::{addr_cmp, Contact, Origin}; @@ -523,9 +523,7 @@ impl Peerstate { let (new_contact_id, _) = Contact::add_or_lookup(context, "", new_addr, Origin::IncomingUnknownFrom) .await?; - if !is_contact_in_chat(context, *chat_id, new_contact_id).await? { - chat::add_to_chat_contacts_table(context, *chat_id, new_contact_id).await?; - } + chat::add_to_chat_contacts_table(context, *chat_id, &[new_contact_id]).await?; context.emit_event(EventType::ChatModified(*chat_id)); } diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 6fbfdddf0..f677ee9d8 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1530,19 +1530,13 @@ async fn create_or_lookup_group( chat_id_blocked = create_blocked; // Create initial member list. - chat::add_to_chat_contacts_table(context, new_chat_id, ContactId::SELF).await?; - if !from_id.is_special() && !chat::is_contact_in_chat(context, new_chat_id, from_id).await? - { - chat::add_to_chat_contacts_table(context, new_chat_id, from_id).await?; - } - for &to_id in to_ids.iter() { - info!(context, "adding to={:?} to chat id={}", to_id, new_chat_id); - if to_id != ContactId::SELF - && !chat::is_contact_in_chat(context, new_chat_id, to_id).await? - { - chat::add_to_chat_contacts_table(context, new_chat_id, to_id).await?; - } + let mut members = vec![ContactId::SELF]; + if !from_id.is_special() { + members.push(from_id); } + members.extend(to_ids); + members.dedup(); + chat::add_to_chat_contacts_table(context, new_chat_id, &members).await?; // once, we have protected-chats explained in UI, we can uncomment the following lines. // ("verified groups" did not add a message anyway) @@ -1698,6 +1692,7 @@ async fn apply_group_changes( .update_timestamp(context, Param::MemberListTimestamp, sent_timestamp) .await? { + let mut members_to_add = vec![]; if removed_id.is_some() || !chat::is_contact_in_chat(context, chat_id, ContactId::SELF).await? { @@ -1712,26 +1707,23 @@ async fn apply_group_changes( ) .await?; - if removed_id != Some(ContactId::SELF) { - chat::add_to_chat_contacts_table(context, chat_id, ContactId::SELF).await?; - } + members_to_add.push(ContactId::SELF); } - if !from_id.is_special() - && from_id != ContactId::SELF - && !chat::is_contact_in_chat(context, chat_id, from_id).await? - && removed_id != Some(from_id) - { - chat::add_to_chat_contacts_table(context, chat_id, from_id).await?; + + if !from_id.is_special() { + members_to_add.push(from_id); } - for &to_id in to_ids.iter() { - if to_id != ContactId::SELF - && !chat::is_contact_in_chat(context, chat_id, to_id).await? - && removed_id != Some(to_id) - { - info!(context, "adding to={:?} to chat id={}", to_id, chat_id); - chat::add_to_chat_contacts_table(context, chat_id, to_id).await?; - } + members_to_add.extend(to_ids); + if let Some(removed_id) = removed_id { + members_to_add.retain(|id| *id != removed_id); } + members_to_add.dedup(); + + info!( + context, + "adding {:?} to chat id={}", members_to_add, chat_id + ); + chat::add_to_chat_contacts_table(context, chat_id, &members_to_add).await?; send_event_chat_modified = true; } } @@ -1883,7 +1875,7 @@ async fn create_or_lookup_mailinglist( ) })?; - chat::add_to_chat_contacts_table(context, chat_id, ContactId::SELF).await?; + chat::add_to_chat_contacts_table(context, chat_id, &[ContactId::SELF]).await?; Ok(Some((chat_id, Blocked::Request))) } else { info!(context, "creating list forbidden by caller"); @@ -2013,9 +2005,7 @@ async fn create_adhoc_group( None, ) .await?; - for &member_id in member_ids.iter() { - chat::add_to_chat_contacts_table(context, new_chat_id, member_id).await?; - } + chat::add_to_chat_contacts_table(context, new_chat_id, member_ids).await?; context.emit_event(EventType::ChatModified(new_chat_id)); @@ -5160,13 +5150,10 @@ Reply from different address chat::add_to_chat_contacts_table( &bob, group_id, - bob.add_or_lookup_contact(&alice1).await.id, - ) - .await?; - chat::add_to_chat_contacts_table( - &bob, - group_id, - Contact::create(&bob, "", "charlie@example.org").await?, + &[ + bob.add_or_lookup_contact(&alice1).await.id, + Contact::create(&bob, "", "charlie@example.org").await?, + ], ) .await?; @@ -5247,7 +5234,7 @@ Reply from different address chat::add_to_chat_contacts_table( &bob, group_id, - bob.add_or_lookup_contact(&alice).await.id, + &[bob.add_or_lookup_contact(&alice).await.id], ) .await?; diff --git a/src/securejoin/bob.rs b/src/securejoin/bob.rs index a0f048b83..9162220ca 100644 --- a/src/securejoin/bob.rs +++ b/src/securejoin/bob.rs @@ -60,7 +60,7 @@ pub(super) async fn start_protocol(context: &Context, invite: QrInvite) -> Resul // TODO: how does this group become usable? let group_chat_id = state.joining_chat_id(context).await?; if !is_contact_in_chat(context, group_chat_id, invite.contact_id()).await? { - chat::add_to_chat_contacts_table(context, group_chat_id, invite.contact_id()) + chat::add_to_chat_contacts_table(context, group_chat_id, &[invite.contact_id()]) .await?; } let msg = stock_str::secure_join_started(context, invite.contact_id()).await; diff --git a/src/sql.rs b/src/sql.rs index d1e1c7e2d..4dacb5daf 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -432,7 +432,7 @@ impl Sql { pub async fn transaction(&self, callback: G) -> Result where H: Send + 'static, - G: Send + 'static + FnOnce(&mut rusqlite::Transaction<'_>) -> Result, + G: Send + FnOnce(&mut rusqlite::Transaction<'_>) -> Result, { let mut conn = self.get_conn().await?; tokio::task::block_in_place(move || { diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index fcb69cbac..0b5a46da8 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -649,6 +649,17 @@ CREATE INDEX smtp_messageid ON imap(rfc724_mid); ) .await?; } + if dbversion < 95 { + sql.execute_migration( + "CREATE TABLE new_chats_contacts (chat_id INTEGER, contact_id INTEGER, UNIQUE(chat_id, contact_id));\ + INSERT OR IGNORE INTO new_chats_contacts SELECT * FROM chats_contacts;\ + DROP TABLE chats_contacts;\ + ALTER TABLE new_chats_contacts RENAME TO chats_contacts;\ + CREATE INDEX chats_contacts_index1 ON chats_contacts (chat_id);\ + CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);", + 95 + ).await?; + } let new_version = sql .get_raw_config_int(VERSION_CFG) From f2c97bda6670e16043c217758760dd62a51ad9bd Mon Sep 17 00:00:00 2001 From: link2xt Date: Tue, 29 Nov 2022 17:21:08 +0000 Subject: [PATCH 20/23] jsonrpc: add message errors to MessageObject --- CHANGELOG.md | 1 + deltachat-jsonrpc/src/api/types/message.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fea7c2fe6..2dc1371d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ### API-Changes - Add Python API to send reactions #3762 +- jsonrpc: add message errors to MessageObject #3788 ### Fixes - Make sure malformed messsages will never block receiving further messages anymore #3771 diff --git a/deltachat-jsonrpc/src/api/types/message.rs b/deltachat-jsonrpc/src/api/types/message.rs index a5899ac06..c7e9cfe8d 100644 --- a/deltachat-jsonrpc/src/api/types/message.rs +++ b/deltachat-jsonrpc/src/api/types/message.rs @@ -34,6 +34,9 @@ pub struct MessageObject { view_type: MessageViewtype, state: u32, + /// An error text, if there is one. + error: Option, + timestamp: i64, sort_timestamp: i64, received_timestamp: i64, @@ -167,6 +170,7 @@ impl MessageObject { .get_state() .to_u32() .ok_or_else(|| anyhow!("state conversion to number failed"))?, + error: message.error(), timestamp: message.get_timestamp(), sort_timestamp: message.get_sort_timestamp(), From 0b60cc83419dfcbda5df004d1ba46b4e96c6a891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 21:01:55 +0000 Subject: [PATCH 21/23] cargo: bump typescript-type-def from 0.5.4 to 0.5.5 Bumps [typescript-type-def](https://github.com/dbeckwith/rust-typescript-type-def) from 0.5.4 to 0.5.5. - [Release notes](https://github.com/dbeckwith/rust-typescript-type-def/releases) - [Changelog](https://github.com/dbeckwith/rust-typescript-type-def/blob/master/CHANGELOG.md) - [Commits](https://github.com/dbeckwith/rust-typescript-type-def/compare/v0.5.4...v0.5.5) --- updated-dependencies: - dependency-name: typescript-type-def dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- deltachat-jsonrpc/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 495355b3e..eab779e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3708,9 +3708,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "typescript-type-def" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1781793d51f116db5bb614f42b42aef2f3fdebe8b2f5f7d00254ed6bb14b0c69" +checksum = "947b91d2fe9ec02a6749b8b645541f16f527e2ea88a60b3f774eca26fd657325" dependencies = [ "serde_json", "typescript-type-def-derive", @@ -3718,9 +3718,9 @@ dependencies = [ [[package]] name = "typescript-type-def-derive" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ac1df09a36962cc2b9404e7fd78cc58c060ecb216c24d666de932e16ff6e539" +checksum = "e5c1bfe689e4067733530495b04959b00f05cd95f038bed59af4fc70b3e26240" dependencies = [ "darling 0.13.4", "ident_case", diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 1677c6c6a..b914c6169 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -23,7 +23,7 @@ async-channel = { version = "1.6.1" } futures = { version = "0.3.25" } serde_json = "1.0.87" yerpc = { version = "^0.3.1", features = ["anyhow_expose"] } -typescript-type-def = { version = "0.5.3", features = ["json_value"] } +typescript-type-def = { version = "0.5.5", features = ["json_value"] } tokio = { version = "1.21.2" } sanitize-filename = "0.4" walkdir = "2.3.2" From 80cefdd1d547997ab00ffea27be356948ba79313 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 21:03:06 +0000 Subject: [PATCH 22/23] cargo: bump reqwest from 0.11.12 to 0.11.13 Bumps [reqwest](https://github.com/seanmonstar/reqwest) from 0.11.12 to 0.11.13. - [Release notes](https://github.com/seanmonstar/reqwest/releases) - [Changelog](https://github.com/seanmonstar/reqwest/blob/master/CHANGELOG.md) - [Commits](https://github.com/seanmonstar/reqwest/compare/v0.11.12...v0.11.13) --- updated-dependencies: - dependency-name: reqwest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 495355b3e..7bb23cbc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2851,9 +2851,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ "base64 0.13.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index d24549389..7d4ee4365 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ textwrap = "0.16.0" async-channel = "1.6.1" futures-lite = "1.12.0" tokio-stream = { version = "0.1.11", features = ["fs"] } -reqwest = { version = "0.11.12", features = ["json"] } +reqwest = { version = "0.11.13", features = ["json"] } async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] } [dev-dependencies] From 46721bcf469943324cceb792555d3cf6dc1f9afb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 21:03:25 +0000 Subject: [PATCH 23/23] cargo: bump regex from 1.6.0 to 1.7.0 Bumps [regex](https://github.com/rust-lang/regex) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.6.0...1.7.0) --- updated-dependencies: - dependency-name: regex dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 495355b3e..647505fbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2819,9 +2819,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" dependencies = [ "aho-corasick", "memchr", diff --git a/Cargo.toml b/Cargo.toml index d24549389..2e391b264 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ quick-xml = "0.23" r2d2 = "0.8" r2d2_sqlite = "0.20" rand = "0.8" -regex = "1.6" +regex = "1.7" rusqlite = { version = "0.27", features = ["sqlcipher"] } rust-hsluv = "0.1" rustyline = { version = "10", optional = true }