Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-27 02:16:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into iroh-v0-29-0

Browse Source
This commit is contained in:
dignifiedquire
2024-12-16 22:32:00 +01:00
parent a908f376a2 1e1e5793dd
commit b65949d643
49 changed files with 1756 additions and 1492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
src/blob.rs
View File

@@ -763,7 +763,6 @@ mod tests {
use fs::File;
use super::*;
use crate::chat::{self, create_group_chat, ProtectionStatus};
use crate::message::{Message, Viewtype};
use crate::test_utils::{self, TestContext};
@@ -1456,38 +1455,4 @@ mod tests {
check_image_size(file_saved, width, height);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_increation_in_blobdir() -> Result<()> {
let t = TestContext::new_alice().await;
let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
let file = t.get_blobdir().join("anyfile.dat");
fs::write(&file, b"bla").await?;
let mut msg = Message::new(Viewtype::File);
msg.set_file(file.to_str().unwrap(), None);
let prepared_id = chat::prepare_msg(&t, chat_id, &mut msg).await?;
assert_eq!(prepared_id, msg.id);
assert!(msg.is_increation());
let msg = Message::load_from_db(&t, prepared_id).await?;
assert!(msg.is_increation());
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_increation_not_blobdir() -> Result<()> {
let t = TestContext::new_alice().await;
let chat_id = create_group_chat(&t, ProtectionStatus::Unprotected, "abc").await?;
assert_ne!(t.get_blobdir().to_str(), t.dir.path().to_str());
let file = t.dir.path().join("anyfile.dat");
fs::write(&file, b"bla").await?;
let mut msg = Message::new(Viewtype::File);
msg.set_file(file.to_str().unwrap(), None);
assert!(chat::prepare_msg(&t, chat_id, &mut msg).await.is_err());
Ok(())
}
}

273
src/chat.rs
View File

@@ -312,7 +312,7 @@ impl ChatId {
/// Create a group or mailinglist raw database record with the given parameters.
/// The function does not add SELF nor checks if the record already exists.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub(crate) async fn create_multiuser_record(
context: &Context,
chattype: Chattype,
@@ -888,7 +888,7 @@ impl ChatId {
_ => {
let blob = msg
.param
.get_blob(Param::File, context, !msg.is_increation())
.get_blob(Param::File, context)
.await?
.context("no file stored in params")?;
msg.param.set(Param::File, blob.as_name());
@@ -2677,26 +2677,13 @@ impl ChatIdBlocked {
}
}
/// Prepares a message for sending.
pub async fn prepare_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
ensure!(
!chat_id.is_special(),
"Cannot prepare message for special chat"
);
let msg_id = prepare_msg_common(context, chat_id, msg, MessageState::OutPreparing).await?;
context.emit_msgs_changed(msg.chat_id, msg.id);
Ok(msg_id)
}
async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
if msg.viewtype == Viewtype::Text || msg.viewtype == Viewtype::VideochatInvitation {
// the caller should check if the message text is empty
} else if msg.viewtype.has_file() {
let mut blob = msg
.param
.get_blob(Param::File, context, !msg.is_increation())
.get_blob(Param::File, context)
.await?
.with_context(|| format!("attachment missing for message of type #{}", msg.viewtype))?;
let send_as_is = msg.viewtype == Viewtype::File;
@@ -2771,13 +2758,92 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<()> {
Ok(())
}
/// Returns whether a contact is in a chat or not.
pub async fn is_contact_in_chat(
context: &Context,
chat_id: ChatId,
contact_id: ContactId,
) -> Result<bool> {
// this function works for group and for normal chats, however, it is more useful
// for group chats.
// ContactId::SELF may be used to check, if the user itself is in a group
// chat (ContactId::SELF is not added to normal chats)
let exists = context
.sql
.exists(
"SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=?;",
(chat_id, contact_id),
)
.await?;
Ok(exists)
}
/// Sends a message object to a chat.
///
/// Sends the event #DC_EVENT_MSGS_CHANGED on success.
/// However, this does not imply, the message really reached the recipient -
/// sending may be delayed eg. due to network problems. However, from your
/// view, you're done with the message. Sooner or later it will find its way.
pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
ensure!(
!chat_id.is_special(),
"chat_id cannot be a special chat: {chat_id}"
);
if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing {
msg.param.remove(Param::GuaranteeE2ee);
msg.param.remove(Param::ForcePlaintext);
msg.update_param(context).await?;
}
// protect all system messages against RTLO attacks
if msg.is_system_message() {
msg.text = sanitize_bidi_characters(&msg.text);
}
if !prepare_send_msg(context, chat_id, msg).await?.is_empty() {
if !msg.hidden {
context.emit_msgs_changed(msg.chat_id, msg.id);
}
if msg.param.exists(Param::SetLatitude) {
context.emit_location_changed(Some(ContactId::SELF)).await?;
}
context.scheduler.interrupt_smtp().await;
}
Ok(msg.id)
}
/// Tries to send a message synchronously.
///
/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the
/// message. If this fails, the jobs remain in the database for later sending.
pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
let rowids = prepare_send_msg(context, chat_id, msg).await?;
if rowids.is_empty() {
return Ok(msg.id);
}
let mut smtp = crate::smtp::Smtp::new();
for rowid in rowids {
send_msg_to_smtp(context, &mut smtp, rowid)
.await
.context("failed to send message, queued for later sending")?;
}
context.emit_msgs_changed(msg.chat_id, msg.id);
Ok(msg.id)
}
/// Prepares a message to be sent out.
async fn prepare_msg_common(
///
/// Returns row ids of the `smtp` table.
async fn prepare_send_msg(
context: &Context,
chat_id: ChatId,
msg: &mut Message,
change_state_to: MessageState,
) -> Result<MsgId> {
) -> Result<Vec<i64>> {
let mut chat = Chat::load_from_db(context, chat_id).await?;
// Check if the chat can be sent to.
@@ -2821,7 +2887,7 @@ async fn prepare_msg_common(
};
// ... then change the MessageState in the message object
msg.state = change_state_to;
msg.state = MessageState::OutPending;
prepare_msg_blob(context, msg).await?;
if !msg.hidden {
@@ -2837,125 +2903,6 @@ async fn prepare_msg_common(
.await?;
msg.chat_id = chat_id;
Ok(msg.id)
}
/// Returns whether a contact is in a chat or not.
pub async fn is_contact_in_chat(
context: &Context,
chat_id: ChatId,
contact_id: ContactId,
) -> Result<bool> {
// this function works for group and for normal chats, however, it is more useful
// for group chats.
// ContactId::SELF may be used to check, if the user itself is in a group
// chat (ContactId::SELF is not added to normal chats)
let exists = context
.sql
.exists(
"SELECT COUNT(*) FROM chats_contacts WHERE chat_id=? AND contact_id=?;",
(chat_id, contact_id),
)
.await?;
Ok(exists)
}
/// Sends a message object to a chat.
///
/// Sends the event #DC_EVENT_MSGS_CHANGED on success.
/// However, this does not imply, the message really reached the recipient -
/// sending may be delayed eg. due to network problems. However, from your
/// view, you're done with the message. Sooner or later it will find its way.
// TODO: Do not allow ChatId to be 0, if prepare_msg had been called
// the caller can get it from msg.chat_id. Forwards would need to
// be fixed for this somehow too.
pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
if chat_id.is_unset() {
let forwards = msg.param.get(Param::PrepForwards);
if let Some(forwards) = forwards {
for forward in forwards.split(' ') {
if let Ok(msg_id) = forward.parse::<u32>().map(MsgId::new) {
if let Ok(mut msg) = Message::load_from_db(context, msg_id).await {
send_msg_inner(context, chat_id, &mut msg).await?;
};
}
}
msg.param.remove(Param::PrepForwards);
msg.update_param(context).await?;
}
return send_msg_inner(context, chat_id, msg).await;
}
if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing {
msg.param.remove(Param::GuaranteeE2ee);
msg.param.remove(Param::ForcePlaintext);
msg.update_param(context).await?;
}
send_msg_inner(context, chat_id, msg).await
}
/// Tries to send a message synchronously.
///
/// Creates jobs in the `smtp` table, then drectly opens an SMTP connection and sends the
/// message. If this fails, the jobs remain in the database for later sending.
pub async fn send_msg_sync(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
let rowids = prepare_send_msg(context, chat_id, msg).await?;
if rowids.is_empty() {
return Ok(msg.id);
}
let mut smtp = crate::smtp::Smtp::new();
for rowid in rowids {
send_msg_to_smtp(context, &mut smtp, rowid)
.await
.context("failed to send message, queued for later sending")?;
}
context.emit_msgs_changed(msg.chat_id, msg.id);
Ok(msg.id)
}
async fn send_msg_inner(context: &Context, chat_id: ChatId, msg: &mut Message) -> Result<MsgId> {
// protect all system messages against RTLO attacks
if msg.is_system_message() {
msg.text = sanitize_bidi_characters(&msg.text);
}
if !prepare_send_msg(context, chat_id, msg).await?.is_empty() {
if !msg.hidden {
context.emit_msgs_changed(msg.chat_id, msg.id);
}
if msg.param.exists(Param::SetLatitude) {
context.emit_location_changed(Some(ContactId::SELF)).await?;
}
context.scheduler.interrupt_smtp().await;
}
Ok(msg.id)
}
/// Returns row ids of the `smtp` table.
async fn prepare_send_msg(
context: &Context,
chat_id: ChatId,
msg: &mut Message,
) -> Result<Vec<i64>> {
// prepare_msg() leaves the message state to OutPreparing, we
// only have to change the state to OutPending in this case.
// Otherwise we still have to prepare the message, which will set
// the state to OutPending.
if msg.state != MessageState::OutPreparing {
// automatically prepare normal messages
prepare_msg_common(context, chat_id, msg, MessageState::OutPending).await?;
} else {
// update message state of separately prepared messages
ensure!(
chat_id.is_unset() || chat_id == msg.chat_id,
"Inconsistent chat ID"
);
message::update_msg_state(context, msg.id, MessageState::OutPending).await?;
}
let row_ids = create_send_msg_jobs(context, msg)
.await
.context("Failed to create send jobs")?;
@@ -4173,8 +4120,6 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
bail!("cannot forward drafts.");
}
let original_param = msg.param.clone();
// we tested a sort of broadcast
// by not marking own forwarded messages as such,
// however, this turned out to be to confusing and unclear.
@@ -4197,33 +4142,13 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
// do not leak data as group names; a default subject is generated by mimefactory
msg.subject = "".to_string();
let new_msg_id: MsgId;
if msg.state == MessageState::OutPreparing {
new_msg_id = chat
.prepare_msg_raw(context, &mut msg, None, curr_timestamp)
.await?;
curr_timestamp += 1;
msg.param = original_param;
msg.id = src_msg_id;
if let Some(old_fwd) = msg.param.get(Param::PrepForwards) {
let new_fwd = format!("{} {}", old_fwd, new_msg_id.to_u32());
msg.param.set(Param::PrepForwards, new_fwd);
} else {
msg.param
.set(Param::PrepForwards, new_msg_id.to_u32().to_string());
}
msg.update_param(context).await?;
} else {
msg.state = MessageState::OutPending;
new_msg_id = chat
.prepare_msg_raw(context, &mut msg, None, curr_timestamp)
.await?;
curr_timestamp += 1;
if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
context.scheduler.interrupt_smtp().await;
}
msg.state = MessageState::OutPending;
let new_msg_id = chat
.prepare_msg_raw(context, &mut msg, None, curr_timestamp)
.await?;
curr_timestamp += 1;
if !create_send_msg_jobs(context, &mut msg).await?.is_empty() {
context.scheduler.interrupt_smtp().await;
}
created_chats.push(chat_id);
created_msgs.push(new_msg_id);
@@ -4519,7 +4444,7 @@ pub(crate) async fn delete_and_reset_all_device_msgs(context: &Context) -> Resul
///
/// For example, it can be a message showing that a member was added to a group.
/// Doesn't fail if the chat doesn't exist.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub(crate) async fn add_info_msg_with_cmd(
context: &Context,
chat_id: ChatId,
@@ -4866,15 +4791,12 @@ mod tests {
assert_eq!(test.text, "hello2".to_string());
assert_eq!(test.state, MessageState::OutDraft);
let id_after_prepare = prepare_msg(&t, *chat_id, &mut msg).await?;
assert_eq!(id_after_prepare, id_after_1st_set);
let test = Message::load_from_db(&t, id_after_prepare).await?;
assert_eq!(test.state, MessageState::OutPreparing);
assert!(!test.hidden); // sent draft must no longer be hidden
let id_after_send = send_msg(&t, *chat_id, &mut msg).await?;
assert_eq!(id_after_send, id_after_1st_set);
let test = Message::load_from_db(&t, id_after_send).await?;
assert!(!test.hidden); // sent draft must no longer be hidden
Ok(())
}
@@ -5626,7 +5548,6 @@ mod tests {
let mut msg = Message::new_text("message text".to_string());
assert!(send_msg(&t, device_chat_id, &mut msg).await.is_err());
assert!(prepare_msg(&t, device_chat_id, &mut msg).await.is_err());
let msg_id = add_device_msg(&t, None, Some(&mut msg)).await.unwrap();
assert!(forward_msgs(&t, &[msg_id], device_chat_id).await.is_err());

1
src/contact.rs
View File

@@ -805,7 +805,6 @@ impl Contact {
}
let mut name = sanitize_name(name);
#[allow(clippy::collapsible_if)]
if origin <= Origin::OutgoingTo {
// The user may accidentally have written to a "noreply" address with another MUA:
if addr.contains("noreply")

3
src/ephemeral.rs
View File

@@ -930,7 +930,6 @@ mod tests {
// Alice sends a text message.
let mut msg = Message::new(Viewtype::Text);
chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
let sent = alice.pop_sent_msg().await;
@@ -957,14 +956,12 @@ mod tests {
// Alice sends message to Bob
let mut msg = Message::new(Viewtype::Text);
chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
let sent = alice.pop_sent_msg().await;
bob.recv_msg(&sent).await;
// Alice sends second message to Bob, with no timer
let mut msg = Message::new(Viewtype::Text);
chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
let sent = alice.pop_sent_msg().await;

78
src/html.rs
View File

@@ -7,6 +7,8 @@
//! `MsgId.get_html()` will return HTML -
//! this allows nice quoting, handling linebreaks properly etc.
use std::mem;
use anyhow::{Context as _, Result};
use base64::Engine as _;
use lettre_email::mime::Mime;
@@ -77,21 +79,26 @@ fn get_mime_multipart_type(ctype: &ParsedContentType) -> MimeMultipartType {
struct HtmlMsgParser {
pub html: String,
pub plain: Option<PlainText>,
pub(crate) msg_html: String,
}
impl HtmlMsgParser {
/// Function takes a raw mime-message string,
/// searches for the main-text part
/// and returns that as parser.html
pub async fn from_bytes(context: &Context, rawmime: &[u8]) -> Result<Self> {
pub async fn from_bytes<'a>(
context: &Context,
rawmime: &'a [u8],
) -> Result<(Self, mailparse::ParsedMail<'a>)> {
let mut parser = HtmlMsgParser {
html: "".to_string(),
plain: None,
msg_html: "".to_string(),
};
let parsedmail = mailparse::parse_mail(rawmime)?;
let parsedmail = mailparse::parse_mail(rawmime).context("Failed to parse mail")?;
parser.collect_texts_recursive(&parsedmail).await?;
parser.collect_texts_recursive(context, &parsedmail).await?;
if parser.html.is_empty() {
if let Some(plain) = &parser.plain {
@@ -100,8 +107,8 @@ impl HtmlMsgParser {
} else {
parser.cid_to_data_recursive(context, &parsedmail).await?;
}
Ok(parser)
parser.html += &mem::take(&mut parser.msg_html);
Ok((parser, parsedmail))
}
/// Function iterates over all mime-parts
@@ -114,12 +121,13 @@ impl HtmlMsgParser {
/// therefore we use the first one.
async fn collect_texts_recursive<'a>(
&'a mut self,
context: &'a Context,
mail: &'a mailparse::ParsedMail<'a>,
) -> Result<()> {
match get_mime_multipart_type(&mail.ctype) {
MimeMultipartType::Multiple => {
for cur_data in &mail.subparts {
Box::pin(self.collect_texts_recursive(cur_data)).await?
Box::pin(self.collect_texts_recursive(context, cur_data)).await?
}
Ok(())
}
@@ -128,8 +136,35 @@ impl HtmlMsgParser {
if raw.is_empty() {
return Ok(());
}
let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?;
Box::pin(self.collect_texts_recursive(&mail)).await
let (parser, mail) = Box::pin(HtmlMsgParser::from_bytes(context, &raw)).await?;
if !parser.html.is_empty() {
let mut text = "\r\n\r\n".to_string();
for h in mail.headers {
let key = h.get_key();
if matches!(
key.to_lowercase().as_str(),
"date"
| "from"
| "sender"
| "reply-to"
| "to"
| "cc"
| "bcc"
| "subject"
) {
text += &format!("{key}: {}\r\n", h.get_value());
}
}
text += "\r\n";
self.msg_html += &PlainText {
text,
flowed: false,
delsp: false,
}
.to_html();
self.msg_html += &parser.html;
}
Ok(())
}
MimeMultipartType::Single => {
let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
@@ -175,14 +210,7 @@ impl HtmlMsgParser {
}
Ok(())
}
MimeMultipartType::Message => {
let raw = mail.get_body_raw()?;
if raw.is_empty() {
return Ok(());
}
let mail = mailparse::parse_mail(&raw).context("failed to parse mail")?;
Box::pin(self.cid_to_data_recursive(context, &mail)).await
}
MimeMultipartType::Message => Ok(()),
MimeMultipartType::Single => {
let mimetype = mail.ctype.mimetype.parse::<Mime>()?;
if mimetype.type_() == mime::IMAGE {
@@ -240,7 +268,7 @@ impl MsgId {
warn!(context, "get_html: parser error: {:#}", err);
Ok(None)
}
Ok(parser) => Ok(Some(parser.html)),
Ok((parser, _)) => Ok(Some(parser.html)),
}
} else {
warn!(context, "get_html: no mime for {}", self);
@@ -274,7 +302,7 @@ mod tests {
async fn test_htmlparse_plain_unspecified() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_plain_unspecified.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert_eq!(
parser.html,
r#"<!DOCTYPE html>
@@ -292,7 +320,7 @@ This message does not have Content-Type nor Subject.<br/>
async fn test_htmlparse_plain_iso88591() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_plain_iso88591.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert_eq!(
parser.html,
r#"<!DOCTYPE html>
@@ -310,7 +338,7 @@ message with a non-UTF-8 encoding: äöüßÄÖÜ<br/>
async fn test_htmlparse_plain_flowed() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_plain_flowed.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert!(parser.plain.unwrap().flowed);
assert_eq!(
parser.html,
@@ -332,7 +360,7 @@ and will be wrapped as usual.<br/>
async fn test_htmlparse_alt_plain() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_alt_plain.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert_eq!(
parser.html,
r#"<!DOCTYPE html>
@@ -353,7 +381,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
async fn test_htmlparse_html() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_html.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
// on windows, `\r\n` linends are returned from mimeparser,
// however, rust multiline-strings use just `\n`;
@@ -371,7 +399,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
async fn test_htmlparse_alt_html() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_alt_html.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert_eq!(
parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
r##"<html>
@@ -386,7 +414,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
async fn test_htmlparse_alt_plain_html() {
let t = TestContext::new().await;
let raw = include_bytes!("../test-data/message/text_alt_plain_html.eml");
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert_eq!(
parser.html.replace('\r', ""), // see comment in test_htmlparse_html()
r##"<html>
@@ -411,7 +439,7 @@ test some special html-characters as &lt; &gt; and &amp; but also &quot; and &#x
assert!(test.find("data:").is_none());
// parsing converts cid: to data:
let parser = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
let (parser, _) = HtmlMsgParser::from_bytes(&t.ctx, raw).await.unwrap();
assert!(parser.html.contains("<html>"));
assert!(!parser.html.contains("Content-Id:"));
assert!(parser.html.contains("data:image/jpeg;base64,/9j/4AAQ"));

87
src/imap.rs
View File

@@ -1301,7 +1301,7 @@ impl Session {
/// Returns the last UID fetched successfully and the info about each downloaded message.
/// If the message is incorrect or there is a failure to write a message to the database,
/// it is skipped and the error is logged.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub(crate) async fn fetch_many_msgs(
&mut self,
context: &Context,
@@ -1560,52 +1560,54 @@ impl Session {
return Ok(());
};
let device_token_changed = context
.get_config(Config::DeviceToken)
.await?
.map_or(true, |config_token| device_token != config_token);
if device_token_changed && self.can_metadata() && self.can_push() {
let folder = context
.get_config(Config::ConfiguredInboxFolder)
if self.can_metadata() && self.can_push() {
let device_token_changed = context
.get_config(Config::DeviceToken)
.await?
.context("INBOX is not configured")?;
.map_or(true, |config_token| device_token != config_token);
let encrypted_device_token =
encrypt_device_token(&device_token).context("Failed to encrypt device token")?;
if device_token_changed {
let folder = context
.get_config(Config::ConfiguredInboxFolder)
.await?
.context("INBOX is not configured")?;
// We expect that the server supporting `XDELTAPUSH` capability
// has non-synchronizing literals support as well:
// <https://www.rfc-editor.org/rfc/rfc7888>.
let encrypted_device_token_len = encrypted_device_token.len();
let encrypted_device_token = encrypt_device_token(&device_token)
.context("Failed to encrypt device token")?;
if encrypted_device_token_len <= 4096 {
self.run_command_and_check_ok(&format_setmetadata(
&folder,
&encrypted_device_token,
))
.await
.context("SETMETADATA command failed")?;
// We expect that the server supporting `XDELTAPUSH` capability
// has non-synchronizing literals support as well:
// <https://www.rfc-editor.org/rfc/rfc7888>.
let encrypted_device_token_len = encrypted_device_token.len();
// Store device token saved on the server
// to prevent storing duplicate tokens.
// The server cannot deduplicate on its own
// because encryption gives a different
// result each time.
context
.set_config_internal(Config::DeviceToken, Some(&device_token))
.await?;
} else {
// If Apple or Google (FCM) gives us a very large token,
// do not even try to give it to IMAP servers.
//
// Limit of 4096 is arbitrarily selected
// to be the same as required by LITERAL- IMAP extension.
//
// Dovecot supports LITERAL+ and non-synchronizing literals
// of any length, but there is no reason for tokens
// to be that large even after OpenPGP encryption.
warn!(context, "Device token is too long for LITERAL-, ignoring.");
if encrypted_device_token_len <= 4096 {
self.run_command_and_check_ok(&format_setmetadata(
&folder,
&encrypted_device_token,
))
.await
.context("SETMETADATA command failed")?;
// Store device token saved on the server
// to prevent storing duplicate tokens.
// The server cannot deduplicate on its own
// because encryption gives a different
// result each time.
context
.set_config_internal(Config::DeviceToken, Some(&device_token))
.await?;
} else {
// If Apple or Google (FCM) gives us a very large token,
// do not even try to give it to IMAP servers.
//
// Limit of 4096 is arbitrarily selected
// to be the same as required by LITERAL- IMAP extension.
//
// Dovecot supports LITERAL+ and non-synchronizing literals
// of any length, but there is no reason for tokens
// to be that large even after OpenPGP encryption.
warn!(context, "Device token is too long for LITERAL-, ignoring.");
}
}
context.push_subscribed.store(true, Ordering::Relaxed);
} else if !context.push_subscriber.heartbeat_subscribed().await {
@@ -2687,7 +2689,6 @@ mod tests {
}
}
#[allow(clippy::too_many_arguments)]
async fn check_target_folder_combination(
folder: &str,
mvbox_move: bool,

64
src/key.rs
View File

@@ -30,7 +30,39 @@ use crate::tools::{self, time_elapsed};
pub(crate) trait DcKey: Serialize + Deserializable + PublicKeyTrait + Clone {
/// Create a key from some bytes.
fn from_slice(bytes: &[u8]) -> Result<Self> {
Ok(<Self as Deserializable>::from_bytes(Cursor::new(bytes))?)
let res = <Self as Deserializable>::from_bytes(Cursor::new(bytes));
if let Ok(res) = res {
return Ok(res);
}
// Workaround for keys imported using
// Delta Chat core < 1.0.0.
// Old Delta Chat core had a bug
// that resulted in treating CRC24 checksum
// as part of the key when reading ASCII Armor.
// Some users that started using Delta Chat in 2019
// have such corrupted keys with garbage bytes at the end.
//
// Garbage is at least 3 bytes long
// and may be longer due to padding
// at the end of the real key data
// and importing the key multiple times.
//
// If removing 10 bytes is not enough,
// the key is likely actually corrupted.
for garbage_bytes in 3..std::cmp::min(bytes.len(), 10) {
let res = <Self as Deserializable>::from_bytes(Cursor::new(
bytes
.get(..bytes.len().saturating_sub(garbage_bytes))
.unwrap_or_default(),
));
if let Ok(res) = res {
return Ok(res);
}
}
// Removing garbage bytes did not help, return the error.
Ok(res?)
}
/// Create a key from a base64 string.
@@ -565,6 +597,36 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
}
}
/// Tests workaround for Delta Chat core < 1.0.0
/// which parsed CRC24 at the end of ASCII Armor
/// as the part of the key.
/// Depending on the alignment and the number of
/// `=` characters at the end of the key,
/// this resulted in various number of garbage
/// octets at the end of the key, starting from 3 octets,
/// but possibly 4 or 5 and maybe more octets
/// if the key is imported or transferred
/// using Autocrypt Setup Message multiple times.
#[test]
fn test_ignore_trailing_garbage() {
// Test several variants of garbage.
for garbage in [
b"\x02\xfc\xaa\x38\x4b\x5c".as_slice(),
b"\x02\xfc\xaa".as_slice(),
b"\x01\x02\x03\x04\x05".as_slice(),
] {
let private_key = KEYPAIR.secret.clone();
let mut binary = DcKey::to_bytes(&private_key);
binary.extend(garbage);
let private_key2 =
SignedSecretKey::from_slice(&binary).expect("Failed to ignore garbage");
assert_eq!(private_key.dc_fingerprint(), private_key2.dc_fingerprint());
}
}
#[test]
fn test_base64_roundtrip() {
let key = KEYPAIR.public.clone();

50
src/message.rs
View File

@@ -348,7 +348,7 @@ impl MsgId {
let server_urls = Self::get_info_server_urls(context, msg.rfc724_mid).await?;
for server_url in server_urls {
// Format as RFC 5092 relative IMAP URL.
ret += &format!("\n{server_url}");
ret += &format!("\nServer-URL: {server_url}");
}
}
let hop_info = self.hop_info(context).await?;
@@ -953,18 +953,6 @@ impl Message {
cmd != SystemMessage::Unknown
}
/// Whether the message is still being created.
///
/// Messages with attachments might be created before the
/// attachment is ready. In this case some more restrictions on
/// the attachment apply, e.g. if the file to be attached is still
/// being written to or otherwise will still change it can not be
/// copied to the blobdir. Thus those attachments need to be
/// created immediately in the blobdir with a valid filename.
pub fn is_increation(&self) -> bool {
self.viewtype.has_file() && self.state == MessageState::OutPreparing
}
/// Returns true if the message is an Autocrypt Setup Message.
pub fn is_setupmessage(&self) -> bool {
if self.viewtype != Viewtype::File {
@@ -2206,38 +2194,6 @@ mod tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_prepare_message_and_send() {
let d = test::TestContext::new().await;
let ctx = &d.ctx;
ctx.set_config(Config::ConfiguredAddr, Some("self@example.com"))
.await
.unwrap();
let chat = d.create_chat_with_contact("", "dest@example.com").await;
let mut msg = Message::new(Viewtype::Text);
let msg_id = chat::prepare_msg(ctx, chat.id, &mut msg).await.unwrap();
let _msg2 = Message::load_from_db(ctx, msg_id).await.unwrap();
assert_eq!(_msg2.get_filemime(), None);
}
/// Tests that message can be prepared even if account has no configured address.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_prepare_not_configured() {
let d = test::TestContext::new().await;
let ctx = &d.ctx;
let chat = d.create_chat_with_contact("", "dest@example.com").await;
let mut msg = Message::new(Viewtype::Text);
assert!(chat::prepare_msg(ctx, chat.id, &mut msg).await.is_ok());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_parse_webrtc_instance() {
let (webrtc_type, url) = Message::parse_webrtc_instance("basicwebrtc:https://foo/bar");
@@ -2357,9 +2313,9 @@ mod tests {
let mut msg = Message::new_text("Quoted message".to_string());
// Prepare message for sending, so it gets a Message-Id.
// Send message, so it gets a Message-Id.
assert!(msg.rfc724_mid.is_empty());
let msg_id = chat::prepare_msg(ctx, chat.id, &mut msg).await.unwrap();
let msg_id = chat::send_msg(ctx, chat.id, &mut msg).await.unwrap();
let msg = Message::load_from_db(ctx, msg_id).await.unwrap();
assert!(!msg.rfc724_mid.is_empty());

21
src/mimefactory.rs
View File

@@ -1047,7 +1047,6 @@ impl MimeFactory {
part.body(text)
}
#[allow(clippy::cognitive_complexity)]
async fn render_message(
&mut self,
context: &Context,
@@ -1516,7 +1515,7 @@ async fn build_body_file(
) -> Result<(PartBuilder, String)> {
let blob = msg
.param
.get_blob(Param::File, context, true)
.get_blob(Param::File, context)
.await?
.context("msg has no file")?;
let suffix = blob.suffix().unwrap_or("dat");
@@ -1905,7 +1904,7 @@ mod tests {
)
.await
.unwrap();
let new_msg = incoming_msg_to_reply_msg(
let mut new_msg = incoming_msg_to_reply_msg(
b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
From: bob@example.com\n\
To: alice@example.org\n\
@@ -1931,6 +1930,9 @@ mod tests {
Original-Message-ID: <2893@example.com>\n\
Disposition: manual-action/MDN-sent-automatically; displayed\n\
\n", &t).await;
chat::send_msg(&t, new_msg.chat_id, &mut new_msg)
.await
.unwrap();
let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
// The subject string should not be "Re: message opened"
assert_eq!("Re: Hello, Bob", mf.subject_str(&t).await.unwrap());
@@ -2077,7 +2079,7 @@ mod tests {
let mut new_msg = Message::new_text("Hi".to_string());
new_msg.chat_id = chat_id;
chat::prepare_msg(&t, chat_id, &mut new_msg).await.unwrap();
chat::send_msg(&t, chat_id, &mut new_msg).await.unwrap();
let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
@@ -2134,7 +2136,7 @@ mod tests {
) -> String {
let t = TestContext::new_alice().await;
let mut new_msg = incoming_msg_to_reply_msg(imf_raw, &t).await;
let incoming_msg = get_chat_msg(&t, new_msg.chat_id, 0, 2).await;
let incoming_msg = get_chat_msg(&t, new_msg.chat_id, 0, 1).await;
if delete_original_msg {
incoming_msg.id.trash(&t, false).await.unwrap();
@@ -2164,6 +2166,9 @@ mod tests {
new_msg.set_quote(&t, Some(&incoming_msg)).await.unwrap();
}
chat::send_msg(&t, new_msg.chat_id, &mut new_msg)
.await
.unwrap();
let mf = MimeFactory::from_msg(&t, new_msg).await.unwrap();
mf.subject_str(&t).await.unwrap()
}
@@ -2184,9 +2189,6 @@ mod tests {
let mut new_msg = Message::new_text("Hi".to_string());
new_msg.chat_id = chat_id;
chat::prepare_msg(context, chat_id, &mut new_msg)
.await
.unwrap();
new_msg
}
@@ -2197,7 +2199,7 @@ mod tests {
let t = TestContext::new_alice().await;
let context = &t;
let msg = incoming_msg_to_reply_msg(
let mut msg = incoming_msg_to_reply_msg(
b"Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
From: Charlie <charlie@example.com>\n\
To: alice@example.org\n\
@@ -2210,6 +2212,7 @@ mod tests {
context,
)
.await;
chat::send_msg(&t, msg.chat_id, &mut msg).await.unwrap();
let mimefactory = MimeFactory::from_msg(&t, msg).await.unwrap();

18
src/mimeparser.rs
View File

@@ -105,14 +105,12 @@ pub(crate) struct MimeMessage {
/// received.
pub(crate) footer: Option<String>,
// if this flag is set, the parts/text/etc. are just close to the original mime-message;
// clients should offer a way to view the original message in this case
/// If set, this is a modified MIME message; clients should offer a way to view the original
/// MIME message in this case.
pub is_mime_modified: bool,
/// The decrypted, raw mime structure.
///
/// This is non-empty iff `is_mime_modified` and the message was actually encrypted. It is used
/// for e.g. late-parsing HTML.
/// Decrypted, raw MIME structure. Nonempty iff `is_mime_modified` and the message was actually
/// encrypted.
pub decoded_data: Vec<u8>,
/// Hop info for debugging.
@@ -1281,7 +1279,7 @@ impl MimeMessage {
Ok(self.parts.len() > old_part_count)
}
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
async fn do_add_single_file_part(
&mut self,
context: &Context,
@@ -3091,11 +3089,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
// Make sure the file is there even though the html is wrong:
let param = &message.parts[0].param;
let blob: BlobObject = param
.get_blob(Param::File, &t, false)
.await
.unwrap()
.unwrap();
let blob: BlobObject = param.get_blob(Param::File, &t).await.unwrap().unwrap();
let f = tokio::fs::File::open(blob.to_abs_path()).await.unwrap();
let size = f.metadata().await.unwrap().len();
assert_eq!(size, 154);

284
src/net/http.rs
View File

@@ -6,14 +6,17 @@ use http_body_util::BodyExt;
use hyper_util::rt::TokioIo;
use mime::Mime;
use serde::Serialize;
use tokio::fs;
use crate::blob::BlobObject;
use crate::context::Context;
use crate::net::proxy::ProxyConfig;
use crate::net::session::SessionStream;
use crate::net::tls::wrap_rustls;
use crate::tools::{create_id, time};
/// HTTP(S) GET response.
#[derive(Debug)]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Response {
/// Response body.
pub blob: Vec<u8>,
@@ -90,9 +93,144 @@ where
Ok(sender)
}
/// Retrieves the binary contents of URL using HTTP GET request.
pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
let mut url = url.to_string();
/// Converts the URL to expiration and stale timestamps.
fn http_url_cache_timestamps(url: &str, mimetype: Option<&str>) -> (i64, i64) {
let now = time();
let expires = now + 3600 * 24 * 35;
let stale = if url.ends_with(".xdc") {
// WebXDCs are never stale, they just expire.
expires
} else if mimetype.is_some_and(|s| s.starts_with("image/")) {
// Cache images for 1 day.
//
// As of 2024-12-12 WebXDC icons at <https://webxdc.org/apps/>
// use the same path for all app versions,
// so may change, but it is not critical if outdated icon is displayed.
now + 3600 * 24
} else {
// Revalidate everything else after 1 hour.
//
// This includes HTML, CSS and JS.
now + 3600
};
(expires, stale)
}
/// Places the binary into HTTP cache.
async fn http_cache_put(context: &Context, url: &str, response: &Response) -> Result<()> {
let blob = BlobObject::create(
context,
&format!("http_cache_{}", create_id()),
response.blob.as_slice(),
)
.await?;
let (expires, stale) = http_url_cache_timestamps(url, response.mimetype.as_deref());
context
.sql
.insert(
"INSERT OR REPLACE INTO http_cache (url, expires, stale, blobname, mimetype, encoding)
VALUES (?, ?, ?, ?, ?, ?)",
(
url,
expires,
stale,
blob.as_name(),
response.mimetype.as_deref().unwrap_or_default(),
response.encoding.as_deref().unwrap_or_default(),
),
)
.await?;
Ok(())
}
/// Retrieves the binary from HTTP cache.
///
/// Also returns if the response is stale and should be revalidated in the background.
async fn http_cache_get(context: &Context, url: &str) -> Result<Option<(Response, bool)>> {
let now = time();
let Some((blob_name, mimetype, encoding, is_stale)) = context
.sql
.query_row_optional(
"SELECT blobname, mimetype, encoding, stale
FROM http_cache WHERE url=? AND expires > ?",
(url, now),
|row| {
let blob_name: String = row.get(0)?;
let mimetype: Option<String> = Some(row.get(1)?).filter(|s: &String| !s.is_empty());
let encoding: Option<String> = Some(row.get(2)?).filter(|s: &String| !s.is_empty());
let stale_timestamp: i64 = row.get(3)?;
Ok((blob_name, mimetype, encoding, now > stale_timestamp))
},
)
.await?
else {
return Ok(None);
};
let blob_object = BlobObject::from_name(context, blob_name)?;
let blob_abs_path = blob_object.to_abs_path();
let blob = match fs::read(blob_abs_path)
.await
.with_context(|| format!("Failed to read blob for {url:?} cache entry."))
{
Ok(blob) => blob,
Err(err) => {
// This should not happen, but user may go into the blobdir and remove files,
// antivirus may delete the file or there may be a bug in housekeeping.
warn!(context, "{err:?}.");
return Ok(None);
}
};
let (expires, _stale) = http_url_cache_timestamps(url, mimetype.as_deref());
let response = Response {
blob,
mimetype,
encoding,
};
// Update expiration timestamp
// to prevent deletion of the file still in use.
//
// We do not update stale timestamp here
// as we have not revalidated the response.
// Stale timestamp is updated only
// when the URL is sucessfully fetched.
context
.sql
.execute(
"UPDATE http_cache SET expires=? WHERE url=?",
(expires, url),
)
.await?;
Ok(Some((response, is_stale)))
}
/// Removes expired cache entries.
pub(crate) async fn http_cache_cleanup(context: &Context) -> Result<()> {
// Remove cache entries that are already expired
// or entries that will not expire in a year
// to make sure we don't have invalid timestamps that are way forward in the future.
context
.sql
.execute(
"DELETE FROM http_cache
WHERE ?1 > expires OR expires > ?1 + 31536000",
(time(),),
)
.await?;
Ok(())
}
/// Fetches URL and updates the cache.
///
/// URL is fetched regardless of whether there is an existing result in the cache.
async fn fetch_url(context: &Context, original_url: &str) -> Result<Response> {
let mut url = original_url.to_string();
// Follow up to 10 http-redirects
for _i in 0..10 {
@@ -139,16 +277,44 @@ pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
});
let body = response.collect().await?.to_bytes();
let blob: Vec<u8> = body.to_vec();
return Ok(Response {
let response = Response {
blob,
mimetype,
encoding,
});
};
info!(context, "Inserting {original_url:?} into cache.");
http_cache_put(context, &url, &response).await?;
return Ok(response);
}
Err(anyhow!("Followed 10 redirections"))
}
/// Retrieves the binary contents of URL using HTTP GET request.
pub async fn read_url_blob(context: &Context, url: &str) -> Result<Response> {
if let Some((response, is_stale)) = http_cache_get(context, url).await? {
info!(context, "Returning {url:?} from cache.");
if is_stale {
let context = context.clone();
let url = url.to_string();
tokio::spawn(async move {
// Fetch URL in background to update the cache.
info!(context, "Fetching stale {url:?} in background.");
if let Err(err) = fetch_url(&context, &url).await {
warn!(context, "Failed to revalidate {url:?}: {err:#}.");
}
});
}
// Return stale result.
return Ok(response);
}
info!(context, "Not found {url:?} in cache, fetching.");
let response = fetch_url(context, url).await?;
Ok(response)
}
/// Sends an empty POST request to the URL.
///
/// Returns response text and whether request was successful or not.
@@ -241,3 +407,109 @@ pub(crate) async fn post_form<T: Serialize + ?Sized>(
let bytes = response.collect().await?.to_bytes();
Ok(bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use crate::sql::housekeeping;
use crate::test_utils::TestContext;
use crate::tools::SystemTime;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_http_cache() -> Result<()> {
let t = &TestContext::new().await;
assert_eq!(http_cache_get(t, "https://webxdc.org/").await?, None);
let html_response = Response {
blob: b"<!DOCTYPE html> ...".to_vec(),
mimetype: Some("text/html".to_string()),
encoding: None,
};
let xdc_response = Response {
blob: b"PK...".to_vec(),
mimetype: Some("application/octet-stream".to_string()),
encoding: None,
};
let xdc_editor_url = "https://apps.testrun.org/webxdc-editor-v3.2.0.xdc";
let xdc_pixel_url = "https://apps.testrun.org/webxdc-pixel-v2.xdc";
http_cache_put(t, "https://webxdc.org/", &html_response).await?;
assert_eq!(http_cache_get(t, xdc_editor_url).await?, None);
assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None);
assert_eq!(
http_cache_get(t, "https://webxdc.org/").await?,
Some((html_response.clone(), false))
);
http_cache_put(t, xdc_editor_url, &xdc_response).await?;
http_cache_put(t, xdc_pixel_url, &xdc_response).await?;
assert_eq!(
http_cache_get(t, xdc_editor_url).await?,
Some((xdc_response.clone(), false))
);
assert_eq!(
http_cache_get(t, xdc_pixel_url).await?,
Some((xdc_response.clone(), false))
);
assert_eq!(
http_cache_get(t, "https://webxdc.org/").await?,
Some((html_response.clone(), false))
);
// HTML is stale after 1 hour, but .xdc is not.
SystemTime::shift(Duration::from_secs(3600 + 100));
assert_eq!(
http_cache_get(t, "https://webxdc.org/").await?,
Some((html_response.clone(), true))
);
assert_eq!(
http_cache_get(t, xdc_editor_url).await?,
Some((xdc_response.clone(), false))
);
// Stale cache entry can be renewed
// even before housekeeping removes old one.
http_cache_put(t, "https://webxdc.org/", &html_response).await?;
assert_eq!(
http_cache_get(t, "https://webxdc.org/").await?,
Some((html_response.clone(), false))
);
// 35 days later pixel .xdc expires because we did not request it for 35 days and 1 hour.
// But editor is still there because we did not request it for just 35 days.
// We have not renewed the editor however, so it becomes stale.
SystemTime::shift(Duration::from_secs(3600 * 24 * 35 - 100));
// Run housekeeping to test that it does not delete the blob too early.
housekeeping(t).await?;
assert_eq!(
http_cache_get(t, xdc_editor_url).await?,
Some((xdc_response.clone(), true))
);
assert_eq!(http_cache_get(t, xdc_pixel_url).await?, None);
// Test that if the file is accidentally removed from the blobdir,
// there is no error when trying to load the cache entry.
for entry in std::fs::read_dir(t.get_blobdir())? {
let entry = entry.unwrap();
let path = entry.path();
std::fs::remove_file(path).expect("Failed to remove blob");
}
assert_eq!(
http_cache_get(t, xdc_editor_url)
.await
.context("Failed to get no cache response")?,
None
);
Ok(())
}
}

23
src/param.rs
View File

@@ -366,20 +366,15 @@ impl Params {
///
/// This parses the parameter value as a [ParamsFile] and than
/// tries to return a [BlobObject] for that file. If the file is
/// not yet a valid blob, one will be created by copying the file
/// only if `create` is set to `true`, otherwise an error is
/// returned.
/// not yet a valid blob, one will be created by copying the file.
///
/// Note that in the [ParamsFile::FsPath] case the blob can be
/// created without copying if the path already refers to a valid
/// blob. If so a [BlobObject] will be returned regardless of the
/// `create` argument.
#[allow(clippy::needless_lifetimes)]
/// blob. If so a [BlobObject] will be returned.
pub async fn get_blob<'a>(
&self,
key: Param,
context: &'a Context,
create: bool,
) -> Result<Option<BlobObject<'a>>> {
let val = match self.get(key) {
Some(val) => val,
@@ -387,10 +382,7 @@ impl Params {
};
let file = ParamsFile::from_param(context, val)?;
let blob = match file {
ParamsFile::FsPath(path) => match create {
true => BlobObject::new_from_path(context, &path).await?,
false => BlobObject::from_path(context, &path)?,
},
ParamsFile::FsPath(path) => BlobObject::new_from_path(context, &path).await?,
ParamsFile::Blob(blob) => blob,
};
Ok(Some(blob))
@@ -546,23 +538,20 @@ mod tests {
let path: PathBuf = p.get_path(Param::File, &t).unwrap().unwrap();
assert_eq!(path, fname);
// Blob does not exist yet, expect error.
assert!(p.get_blob(Param::File, &t, false).await.is_err());
fs::write(fname, b"boo").await.unwrap();
let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap();
let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap();
assert!(blob.as_file_name().starts_with("foo"));
// Blob in blobdir, expect blob.
let bar_path = t.get_blobdir().join("bar");
p.set(Param::File, bar_path.to_str().unwrap());
let blob = p.get_blob(Param::File, &t, false).await.unwrap().unwrap();
let blob = p.get_blob(Param::File, &t).await.unwrap().unwrap();
assert_eq!(blob, BlobObject::from_name(&t, "bar".to_string()).unwrap());
p.remove(Param::File);
assert!(p.get_file(Param::File, &t).unwrap().is_none());
assert!(p.get_path(Param::File, &t).unwrap().is_none());
assert!(p.get_blob(Param::File, &t, false).await.unwrap().is_none());
assert!(p.get_blob(Param::File, &t).await.unwrap().is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

2
src/pgp.rs
View File

@@ -21,11 +21,9 @@ use tokio::runtime::Handle;
use crate::constants::KeyGenType;
use crate::key::{DcKey, Fingerprint};
#[allow(missing_docs)]
#[cfg(test)]
pub(crate) const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
#[allow(missing_docs)]
pub const HEADER_SETUPCODE: &str = "passphrase-begin";
/// Preferred symmetric encryption algorithm.

2
src/quota.rs
View File

@@ -187,7 +187,7 @@ mod tests {
Ok(())
}
#[allow(clippy::assertions_on_constants)]
#[expect(clippy::assertions_on_constants)]
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_quota_thresholds() -> anyhow::Result<()> {
assert!(QUOTA_ALLCLEAR_PERCENTAGE > 50);

35
src/receive_imf.rs
View File

@@ -158,7 +158,7 @@ async fn insert_tombstone(context: &Context, rfc724_mid: &str) -> Result<MsgId>
/// If `is_partial_download` is set, it contains the full message size in bytes.
/// Do not confuse that with `replace_msg_id` that will be set when the full message is loaded
/// later.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub(crate) async fn receive_imf_inner(
context: &Context,
folder: &str,
@@ -679,7 +679,7 @@ pub async fn from_field_to_contact_id(
/// Creates a `ReceivedMsg` from given parts which might consist of
/// multiple messages (if there are multiple attachments).
/// Every entry in `mime_parser.parts` produces a new row in the `msgs` table.
#[allow(clippy::too_many_arguments, clippy::cognitive_complexity)]
#[expect(clippy::too_many_arguments)]
async fn add_parts(
context: &Context,
mime_parser: &mut MimeMessage,
@@ -1406,10 +1406,11 @@ async fn add_parts(
// we save the full mime-message and add a flag
// that the ui should show button to display the full message.
// a flag used to avoid adding "show full message" button to multiple parts of the message.
let mut save_mime_modified = mime_parser.is_mime_modified;
// We add "Show Full Message" button to the last message bubble (part) if this flag evaluates to
// `true` finally.
let mut save_mime_modified = false;
let mime_headers = if save_mime_headers || save_mime_modified {
let mime_headers = if save_mime_headers || mime_parser.is_mime_modified {
let headers = if !mime_parser.decoded_data.is_empty() {
mime_parser.decoded_data.clone()
} else {
@@ -1475,7 +1476,8 @@ async fn add_parts(
}
}
for part in &mime_parser.parts {
let mut parts = mime_parser.parts.iter().peekable();
while let Some(part) = parts.next() {
if part.is_reaction {
let reaction_str = simplify::remove_footers(part.msg.as_str());
let is_incoming_fresh = mime_parser.incoming && !seen && !fetching_existing_messages;
@@ -1519,14 +1521,11 @@ async fn add_parts(
} else {
(&part.msg, part.typ)
};
let part_is_empty =
typ == Viewtype::Text && msg.is_empty() && part.param.get(Param::Quote).is_none();
let mime_modified = save_mime_modified && !part_is_empty;
if mime_modified {
// Avoid setting mime_modified for more than one part.
save_mime_modified = false;
}
save_mime_modified |= mime_parser.is_mime_modified && !part_is_empty && !hidden;
let save_mime_modified = save_mime_modified && parts.peek().is_none();
if part.typ == Viewtype::Text {
let msg_raw = part.msg_raw.as_ref().cloned().unwrap_or_default();
@@ -1546,8 +1545,7 @@ async fn add_parts(
// If you change which information is skipped if the message is trashed,
// also change `MsgId::trash()` and `delete_expired_messages()`
let trash =
chat_id.is_trash() || (is_location_kml && msg.is_empty() && typ == Viewtype::Text);
let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified);
let row_id = context
.sql
@@ -1610,14 +1608,14 @@ RETURNING id
},
hidden,
part.bytes as isize,
if (save_mime_headers || mime_modified) && !trash {
if (save_mime_headers || save_mime_modified) && !trash {
mime_headers.clone()
} else {
Vec::new()
},
mime_in_reply_to,
mime_references,
mime_modified,
save_mime_modified,
part.error.as_deref().unwrap_or_default(),
ephemeral_timer,
ephemeral_timestamp,
@@ -1843,7 +1841,7 @@ async fn lookup_chat_by_reply(
Ok(Some((parent_chat.id, parent_chat.blocked)))
}
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
async fn lookup_chat_or_create_adhoc_group(
context: &Context,
mime_parser: &MimeMessage,
@@ -1978,7 +1976,7 @@ async fn is_probably_private_reply(
/// than two members, a new ad hoc group is created.
///
/// On success the function returns the created (chat_id, chat_blocked) tuple.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
async fn create_group(
context: &Context,
mime_parser: &mut MimeMessage,
@@ -2098,7 +2096,6 @@ async fn create_group(
/// just omitted.
///
/// * `is_partial_download` - whether the message is not fully downloaded.
#[allow(clippy::too_many_arguments)]
async fn apply_group_changes(
context: &Context,
mime_parser: &mut MimeMessage,

55
src/receive_imf/tests.rs
View File

@@ -3827,6 +3827,61 @@ async fn test_messed_up_message_id() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_big_forwarded_with_big_attachment() -> Result<()> {
let t = &TestContext::new_bob().await;
let raw = include_bytes!("../../test-data/message/big_forwarded_with_big_attachment.eml");
let rcvd = receive_imf(t, raw, false).await?.unwrap();
assert_eq!(rcvd.msg_ids.len(), 3);
let msg = Message::load_from_db(t, rcvd.msg_ids[0]).await?;
assert_eq!(msg.get_viewtype(), Viewtype::Text);
assert_eq!(msg.get_text(), "Hello!");
assert!(!msg.has_html());
let msg = Message::load_from_db(t, rcvd.msg_ids[1]).await?;
assert_eq!(msg.get_viewtype(), Viewtype::Text);
assert!(msg
.get_text()
.starts_with("this text with 42 chars is just repeated."));
assert!(msg.get_text().ends_with("[...]"));
assert!(!msg.has_html());
let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?;
assert_eq!(msg.get_viewtype(), Viewtype::File);
assert!(msg.has_html());
let html = msg.id.get_html(t).await?.unwrap();
let tail = html
.split_once("Hello!")
.unwrap()
.1
.split_once("From: AAA")
.unwrap()
.1
.split_once("aaa@example.org")
.unwrap()
.1
.split_once("To: Alice")
.unwrap()
.1
.split_once("alice@example.org")
.unwrap()
.1
.split_once("Subject: Some subject")
.unwrap()
.1
.split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000")
.unwrap()
.1;
assert_eq!(
tail.matches("this text with 42 chars is just repeated.")
.count(),
128
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_mua_user_adds_member() -> Result<()> {
let t = TestContext::new_alice().await;

2
src/smtp.rs
View File

@@ -104,7 +104,7 @@ impl Smtp {
}
/// Connect using the provided login params.
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub async fn connect(
&mut self,
context: &Context,

2
src/smtp/connect.rs
View File

@@ -45,7 +45,7 @@ async fn new_smtp_transport<S: AsyncBufRead + AsyncWrite + Unpin>(
Ok(transport)
}
#[allow(clippy::too_many_arguments)]
#[expect(clippy::too_many_arguments)]
pub(crate) async fn connect_and_auth(
context: &Context,
proxy_config: &Option<ProxyConfig>,

24
src/sql.rs
View File

@@ -19,6 +19,7 @@ use crate::location::delete_orphaned_poi_locations;
use crate::log::LogExt;
use crate::message::{Message, MsgId};
use crate::net::dns::prune_dns_cache;
use crate::net::http::http_cache_cleanup;
use crate::net::prune_connection_history;
use crate::param::{Param, Params};
use crate::peerstate::Peerstate;
@@ -720,6 +721,12 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
warn!(context, "Can't set config: {e:#}.");
}
http_cache_cleanup(context)
.await
.context("Failed to cleanup HTTP cache")
.log_err(context)
.ok();
if let Err(err) = remove_unused_files(context).await {
warn!(
context,
@@ -846,6 +853,22 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> {
.await
.context("housekeeping: failed to SELECT value FROM config")?;
context
.sql
.query_map(
"SELECT blobname FROM http_cache",
(),
|row| row.get::<_, String>(0),
|rows| {
for row in rows {
maybe_add_file(&mut files_in_use, &row?);
}
Ok(())
},
)
.await
.context("Failed to SELECT blobname FROM http_cache")?;
info!(context, "{} files in use.", files_in_use.len());
/* go through directories and delete unused files */
let blobdir = context.get_blobdir();
@@ -864,7 +887,6 @@ pub async fn remove_unused_files(context: &Context) -> Result<()> {
if p == blobdir
&& (is_file_in_use(&files_in_use, None, &name_s)
|| is_file_in_use(&files_in_use, Some(".increation"), &name_s)
|| is_file_in_use(&files_in_use, Some(".waveform"), &name_s)
|| is_file_in_use(&files_in_use, Some("-preview.jpg"), &name_s))
{

33
src/sql/migrations.rs
View File

@@ -1088,6 +1088,39 @@ CREATE INDEX msgs_status_updates_index2 ON msgs_status_updates (uid);
.await?;
}
inc_and_check(&mut migration_version, 125)?;
if dbversion < migration_version {
sql.execute_migration(
"CREATE TABLE http_cache (
url TEXT PRIMARY KEY,
expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds.
blobname TEXT NOT NULL,
mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header.
encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header.
) STRICT",
migration_version,
)
.await?;
}
inc_and_check(&mut migration_version, 126)?;
if dbversion < migration_version {
// Recreate http_cache table with new `stale` column.
sql.execute_migration(
"DROP TABLE http_cache;
CREATE TABLE http_cache (
url TEXT PRIMARY KEY,
expires INTEGER NOT NULL, -- When the cache entry is considered expired, timestamp in seconds.
stale INTEGER NOT NULL, -- When the cache entry is considered stale, timestamp in seconds.
blobname TEXT NOT NULL,
mimetype TEXT NOT NULL DEFAULT '', -- MIME type extracted from Content-Type header.
encoding TEXT NOT NULL DEFAULT '' -- Encoding from Content-Type header.
) STRICT",
migration_version,
)
.await?;
}
let new_version = sql
.get_raw_config_int(VERSION_CFG)
.await?