Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-23 00:16:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: message previews

Browse Source 
- Remove partial downloads (remove creation of the stub messages) (#7373)
- Remove "Download maximum available until" and remove stock string `DC_STR_DOWNLOAD_AVAILABILITY` (#7369)
- Send pre-message on messages with large attachments (#7410)
- Pre messages can now get read receipts (#7433)

Co-authored-by: Hocuri <hocuri@gmx.de>
This commit is contained in:
Simon Laux
2025-10-29 21:50:58 +01:00
committed by link2xt
parent 5925f72316
commit a98fe05e08
41 changed files with 2413 additions and 1448 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
deltachat-ffi/deltachat.h
View File

@@ -488,11 +488,11 @@ char* dc_get_blobdir (const dc_context_t* context);
* 0=use IMAP IDLE if the server supports it.
* This is a developer option used for testing polling used as an IDLE fallback.
* - `download_limit` = Messages up to this number of bytes are downloaded automatically.
* For larger messages, only the header is downloaded and a placeholder is shown.
* For messages with large attachments, two messages are sent:
* a Pre-Message containing metadata and a Post-Message containing the attachment.
* Pre-Messages are always downloaded and show a placeholder message.
* These messages can be downloaded fully using dc_download_full_msg() later.
* The limit is compared against raw message sizes, including headers.
* The actually used limit may be corrected
* to not mess up with non-delivery-reports or read-receipts.
* Post-Messages are automatically downloaded if they are smaller than the download_limit.
* 0=no limit (default).
* Changes affect future messages only.
* - `protect_autocrypt` = Enable Header Protection for Autocrypt header.
@@ -4310,7 +4310,8 @@ char* dc_msg_get_webxdc_info (const dc_msg_t* msg);
/**
* Get the size of the file. Returns the size of the file associated with a
* message, if applicable.
* message, if applicable.
* If message is a pre-message, then this returns size of the to be downloaded file.
*
* Typically, this is used to show the size of document files, e.g. a PDF.
*
@@ -7263,22 +7264,9 @@ void dc_event_unref(dc_event_t* event);
/// `%1$s` will be replaced by the percentage used
#define DC_STR_QUOTA_EXCEEDING_MSG_BODY 98
/// "%1$s message"
///
/// Used as the message body when a message
/// was not yet downloaded completely
/// (dc_msg_get_download_state() is e.g. @ref DC_DOWNLOAD_AVAILABLE).
///
/// `%1$s` will be replaced by human-readable size (e.g. "1.2 MiB").
/// @deprecated Deprecated 2025-11-12, this string is no longer needed.
#define DC_STR_PARTIAL_DOWNLOAD_MSG_BODY 99
/// "Download maximum available until %1$s"
///
/// Appended after some separator to @ref DC_STR_PARTIAL_DOWNLOAD_MSG_BODY.
///
/// `%1$s` will be replaced by human-readable date and time.
#define DC_STR_DOWNLOAD_AVAILABILITY 100
/// "Multi Device Synchronization"
///
/// Used in subjects of outgoing sync messages.
@@ -7776,6 +7764,11 @@ void dc_event_unref(dc_event_t* event);
/// Used as the first info messages in newly created classic email threads.
#define DC_STR_CHAT_UNENCRYPTED_EXPLANATON 230
/// "Contact"
///
/// Used in summaries.
#define DC_STR_CONTACT 231
/**
* @}
*/

3
deltachat-jsonrpc/src/api/types/message.rs
View File

@@ -92,6 +92,9 @@ pub struct MessageObject {
file: Option<String>,
file_mime: Option<String>,
/// The size of the file in bytes, if applicable.
/// If message is a pre-message, then this is the size of the to be downloaded file.
file_bytes: u64,
file_name: Option<String>,

4
deltachat-rpc-client/tests/test_chatlist_events.py
View File

@@ -1,7 +1,5 @@
from __future__ import annotations
import base64
import os
from typing import TYPE_CHECKING
from deltachat_rpc_client import Account, EventType, const
@@ -129,7 +127,7 @@ def test_download_on_demand(acfactory: ACFactory) -> None:
msg.get_snapshot().chat.accept()
bob.get_chat_by_id(chat_id).send_message(
"Hello World, this message is bigger than 5 bytes",
html=base64.b64encode(os.urandom(300000)).decode("utf-8"),
file="../test-data/image/screenshot.jpg",
)
message = alice.wait_for_incoming_msg()

100
deltachat-rpc-client/tests/test_something.py
View File

@@ -5,13 +5,12 @@ import logging
import os
import socket
import subprocess
import time
from unittest.mock import MagicMock
import pytest
from deltachat_rpc_client import Contact, EventType, Message, events
from deltachat_rpc_client.const import DownloadState, MessageState
from deltachat_rpc_client import EventType, events
from deltachat_rpc_client.const import MessageState
from deltachat_rpc_client.pytestplugin import E2EE_INFO_MSGS
from deltachat_rpc_client.rpc import JsonRpcError
@@ -333,7 +332,7 @@ def test_receive_imf_failure(acfactory) -> None:
alice_contact_bob = alice.create_contact(bob, "Bob")
alice_chat_bob = alice_contact_bob.create_chat()
bob.set_config("fail_on_receiving_full_msg", "1")
bob.set_config("simulate_receive_imf_error", "1")
alice_chat_bob.send_text("Hello!")
event = bob.wait_for_event(EventType.MSGS_CHANGED)
assert event.chat_id == bob.get_device_chat().id
@@ -342,17 +341,16 @@ def test_receive_imf_failure(acfactory) -> None:
snapshot = message.get_snapshot()
assert (
snapshot.text == "❌ Failed to receive a message:"
" Condition failed: `!context.get_config_bool(Config::FailOnReceivingFullMsg).await?`."
" Condition failed: `!context.get_config_bool(Config::SimulateReceiveImfError).await?`."
" Please report this bug to delta@merlinux.eu or https://support.delta.chat/."
)
# The failed message doesn't break the IMAP loop.
bob.set_config("fail_on_receiving_full_msg", "0")
bob.set_config("simulate_receive_imf_error", "0")
alice_chat_bob.send_text("Hello again!")
message = bob.wait_for_incoming_msg()
snapshot = message.get_snapshot()
assert snapshot.text == "Hello again!"
assert snapshot.download_state == DownloadState.DONE
assert snapshot.error is None
@@ -588,94 +586,6 @@ def test_mdn_doesnt_break_autocrypt(acfactory) -> None:
assert snapshot.show_padlock
def test_reaction_to_partially_fetched_msg(acfactory, tmp_path):
"""See https://github.com/deltachat/deltachat-core-rust/issues/3688 "Partially downloaded
messages are received out of order".
If the Inbox contains X small messages followed by Y large messages followed by Z small
messages, Delta Chat first downloaded a batch of X+Z messages, and then a batch of Y messages.
This bug was discovered by @Simon-Laux while testing reactions PR #3644 and can be reproduced
with online test as follows:
- Bob enables download limit and goes offline.
- Alice sends a large message to Bob and reacts to this message with a thumbs-up.
- Bob goes online
- Bob first processes a reaction message and throws it away because there is no corresponding
message, then processes a partially downloaded message.
- As a result, Bob does not see a reaction
"""
download_limit = 300000
ac1, ac2 = acfactory.get_online_accounts(2)
ac1_addr = ac1.get_config("addr")
chat = ac1.create_chat(ac2)
ac2.set_config("download_limit", str(download_limit))
ac2.stop_io()
logging.info("sending small+large messages from ac1 to ac2")
msgs = []
msgs.append(chat.send_text("hi"))
path = tmp_path / "large"
path.write_bytes(os.urandom(download_limit + 1))
msgs.append(chat.send_file(str(path)))
for m in msgs:
m.wait_until_delivered()
logging.info("sending a reaction to the large message from ac1 to ac2")
# TODO: Find the reason of an occasional message reordering on the server (so that the reaction
# has a lower UID than the previous message). W/a is to sleep for some time to let the reaction
# have a later INTERNALDATE.
time.sleep(1.1)
react_str = "\N{THUMBS UP SIGN}"
msgs.append(msgs[-1].send_reaction(react_str))
msgs[-1].wait_until_delivered()
ac2.start_io()
logging.info("wait for ac2 to receive a reaction")
msg2 = Message(ac2, ac2.wait_for_reactions_changed().msg_id)
assert msg2.get_sender_contact().get_snapshot().address == ac1_addr
assert msg2.get_snapshot().download_state == DownloadState.AVAILABLE
reactions = msg2.get_reactions()
contacts = [Contact(ac2, int(i)) for i in reactions.reactions_by_contact]
assert len(contacts) == 1
assert contacts[0].get_snapshot().address == ac1_addr
assert list(reactions.reactions_by_contact.values())[0] == [react_str]
@pytest.mark.parametrize("n_accounts", [3, 2])
def test_download_limit_chat_assignment(acfactory, tmp_path, n_accounts):
download_limit = 300000
alice, *others = acfactory.get_online_accounts(n_accounts)
bob = others[0]
alice_group = alice.create_group("test group")
for account in others:
chat = account.create_chat(alice)
chat.send_text("Hello Alice!")
assert alice.wait_for_incoming_msg().get_snapshot().text == "Hello Alice!"
contact = alice.create_contact(account)
alice_group.add_contact(contact)
bob.set_config("download_limit", str(download_limit))
alice_group.send_text("hi")
snapshot = bob.wait_for_incoming_msg().get_snapshot()
assert snapshot.text == "hi"
bob_group = snapshot.chat
path = tmp_path / "large"
path.write_bytes(os.urandom(download_limit + 1))
for i in range(10):
logging.info("Sending message %s", i)
alice_group.send_file(str(path))
snapshot = bob.wait_for_incoming_msg().get_snapshot()
assert snapshot.download_state == DownloadState.AVAILABLE
assert snapshot.chat == bob_group
def test_markseen_contact_request(acfactory):
"""
Test that seen status is synchronized for contact request messages

33
python/tests/test_1_online.py
View File

@@ -1,7 +1,6 @@
import os
import queue
import sys
import base64
from datetime import datetime, timezone
import pytest
@@ -222,38 +221,6 @@ def test_webxdc_huge_update(acfactory, data, lp):
assert update["payload"] == payload
def test_webxdc_download_on_demand(acfactory, data, lp):
ac1, ac2 = acfactory.get_online_accounts(2)
acfactory.introduce_each_other([ac1, ac2])
chat = acfactory.get_accepted_chat(ac1, ac2)
msg1 = Message.new_empty(ac1, "webxdc")
msg1.set_text("message1")
msg1.set_file(data.get_path("webxdc/minimal.xdc"))
msg1 = chat.send_msg(msg1)
assert msg1.is_webxdc()
assert msg1.filename
msg2 = ac2._evtracker.wait_next_incoming_message()
assert msg2.is_webxdc()
lp.sec("ac2 sets download limit")
ac2.set_config("download_limit", "100")
assert msg1.send_status_update({"payload": base64.b64encode(os.urandom(300000))}, "some test data")
ac2_update = ac2._evtracker.wait_next_incoming_message()
assert ac2_update.download_state == dc.const.DC_DOWNLOAD_AVAILABLE
assert not msg2.get_status_updates()
ac2_update.download_full()
ac2._evtracker.get_matching("DC_EVENT_WEBXDC_STATUS_UPDATE")
assert msg2.get_status_updates()
# Get a event notifying that the message disappeared from the chat.
msgs_changed_event = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
assert msgs_changed_event.data1 == msg2.chat.id
assert msgs_changed_event.data2 == 0
def test_enable_mvbox_move(acfactory, lp):
(ac1,) = acfactory.get_online_accounts(1)

64
src/calls/calls_tests.rs
View File

@@ -2,7 +2,7 @@ use super::*;
use crate::chat::forward_msgs;
use crate::config::Config;
use crate::constants::DC_CHAT_ID_TRASH;
use crate::receive_imf::{receive_imf, receive_imf_from_inbox};
use crate::receive_imf::receive_imf;
use crate::test_utils::{TestContext, TestContextManager};
struct CallSetup {
@@ -610,65 +610,3 @@ async fn test_end_text_call() -> Result<()> {
Ok(())
}
/// Tests that partially downloaded "call ended"
/// messages are not processed.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_no_partial_calls() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let seen = false;
// The messages in the test
// have no `Date` on purpose,
// so they are treated as new.
let received_call = receive_imf(
alice,
b"From: bob@example.net\n\
To: alice@example.org\n\
Message-ID: <first@example.net>\n\
Chat-Version: 1.0\n\
Chat-Content: call\n\
Chat-Webrtc-Room: YWFhYWFhYWFhCg==\n\
\n\
Hello, this is a call\n",
seen,
)
.await?
.unwrap();
assert_eq!(received_call.msg_ids.len(), 1);
let call_msg = Message::load_from_db(alice, received_call.msg_ids[0])
.await
.unwrap();
assert_eq!(call_msg.viewtype, Viewtype::Call);
assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting);
let imf_raw = b"From: bob@example.net\n\
To: alice@example.org\n\
Message-ID: <second@example.net>\n\
In-Reply-To: <first@example.net>\n\
Chat-Version: 1.0\n\
Chat-Content: call-ended\n\
\n\
Call ended\n";
receive_imf_from_inbox(
alice,
"second@example.net",
imf_raw,
seen,
Some(imf_raw.len().try_into().unwrap()),
)
.await?;
// The call is still not ended.
assert_eq!(call_state(alice, call_msg.id).await?, CallState::Alerting);
// Fully downloading the message ends the call.
receive_imf_from_inbox(alice, "second@example.net", imf_raw, seen, None)
.await
.context("Failed to fully download end call message")?;
assert_eq!(call_state(alice, call_msg.id).await?, CallState::Missed);
Ok(())
}

131
src/chat.rs
View File

@@ -12,6 +12,7 @@ use std::time::Duration;
use anyhow::{Context as _, Result, anyhow, bail, ensure};
use chrono::TimeZone;
use deltachat_contact_tools::{ContactAddress, sanitize_bidi_characters, sanitize_single_line};
use humansize::{BINARY, format_size};
use mail_builder::mime::MimePart;
use serde::{Deserialize, Serialize};
use strum_macros::EnumIter;
@@ -27,7 +28,9 @@ use crate::constants::{
use crate::contact::{self, Contact, ContactId, Origin};
use crate::context::Context;
use crate::debug_logging::maybe_set_logging_xdc;
use crate::download::DownloadState;
use crate::download::{
DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, PRE_MSG_SIZE_WARNING_THRESHOLD,
};
use crate::ephemeral::{Timer as EphemeralTimer, start_chat_ephemeral_timers};
use crate::events::EventType;
use crate::key::self_fingerprint;
@@ -35,7 +38,7 @@ use crate::location;
use crate::log::{LogExt, warn};
use crate::logged_debug_assert;
use crate::message::{self, Message, MessageState, MsgId, Viewtype};
use crate::mimefactory::MimeFactory;
use crate::mimefactory::{MimeFactory, RenderedEmail};
use crate::mimeparser::SystemMessage;
use crate::param::{Param, Params};
use crate::receive_imf::ReceivedMsg;
@@ -2728,6 +2731,57 @@ async fn prepare_send_msg(
Ok(row_ids)
}
/// Renders the Message or splits it into Post-Message and Pre-Message.
///
/// Pre-Message is a small message with metadata which announces a larger Post-Message.
/// Post-Messages are not downloaded in the background.
///
/// If pre-message is not nessesary this returns a normal message instead.
async fn render_mime_message_and_pre_message(
context: &Context,
msg: &mut Message,
mimefactory: MimeFactory,
) -> Result<(RenderedEmail, Option<RenderedEmail>)> {
let needs_pre_message = msg.viewtype.has_file()
&& mimefactory.will_be_encrypted() // unencrypted is likely email, we don't want to spam by sending multiple messages
&& msg
.get_filebytes(context)
.await?
.context("filebytes not available, even though message has attachment")?
> PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
if needs_pre_message {
info!(
context,
"Message is large and will be split into a pre- and a post-message.",
);
let mut mimefactory_post_msg = mimefactory.clone();
mimefactory_post_msg.set_as_post_message();
let rendered_msg = mimefactory_post_msg.render(context).await?;
let mut mimefactory_pre_msg = mimefactory;
mimefactory_pre_msg.set_as_pre_message_for(&rendered_msg);
let rendered_pre_msg = mimefactory_pre_msg
.render(context)
.await
.context("pre-message failed to render")?;
if rendered_pre_msg.message.len() > PRE_MSG_SIZE_WARNING_THRESHOLD {
warn!(
context,
"Pre-message for message (MsgId={}) is larger than expected: {}.",
msg.id,
rendered_pre_msg.message.len()
);
}
Ok((rendered_msg, Some(rendered_pre_msg)))
} else {
Ok((mimefactory.render(context).await?, None))
}
}
/// Constructs jobs for sending a message and inserts them into the appropriate table.
///
/// Updates the message `GuaranteeE2ee` parameter and persists it
@@ -2799,13 +2853,29 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
return Ok(Vec::new());
}
let rendered_msg = match mimefactory.render(context).await {
Ok(res) => Ok(res),
Err(err) => {
message::set_msg_failed(context, msg, &err.to_string()).await?;
Err(err)
}
}?;
let (rendered_msg, rendered_pre_msg) =
match render_mime_message_and_pre_message(context, msg, mimefactory).await {
Ok(res) => Ok(res),
Err(err) => {
message::set_msg_failed(context, msg, &err.to_string()).await?;
Err(err)
}
}?;
if let (post_msg, Some(pre_msg)) = (&rendered_msg, &rendered_pre_msg) {
info!(
context,
"Message Sizes: Pre-Message {}; Post-Message: {}",
format_size(pre_msg.message.len(), BINARY),
format_size(post_msg.message.len(), BINARY)
);
} else {
info!(
context,
"Message will be sent as normal message (no pre- and post message). Size: {}",
format_size(rendered_msg.message.len(), BINARY)
);
}
if needs_encryption && !rendered_msg.is_encrypted {
/* unrecoverable */
@@ -2864,12 +2934,26 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
} else {
for recipients_chunk in recipients.chunks(chunk_size) {
let recipients_chunk = recipients_chunk.join(" ");
// send pre-message before actual message
if let Some(pre_msg) = &rendered_pre_msg {
let row_id = t.execute(
"INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id)
VALUES (?1, ?2, ?3, ?4)",
(
&pre_msg.rfc724_mid,
&recipients_chunk,
&pre_msg.message,
msg.id,
),
)?;
row_ids.push(row_id.try_into()?);
}
let row_id = t.execute(
"INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id) \
"INSERT INTO smtp (rfc724_mid, recipients, mime, msg_id)
VALUES (?1, ?2, ?3, ?4)",
(
&rendered_msg.rfc724_mid,
recipients_chunk,
&recipients_chunk,
&rendered_msg.message,
msg.id,
),
@@ -4256,6 +4340,14 @@ pub async fn forward_msgs_2ctx(
msg.viewtype = Viewtype::Text;
}
if msg.download_state != DownloadState::Done {
// we don't use Message.get_text() here,
// because it may change in future,
// when UI shows this info itself,
// then the additional_text will not be added in get_text anymore.
msg.text += &msg.additional_text;
}
let param = &mut param;
msg.param.steal(param, Param::File);
msg.param.steal(param, Param::Filename);
@@ -4332,12 +4424,22 @@ pub(crate) async fn save_copy_in_self_talk(
msg.param.remove(Param::WebxdcDocumentTimestamp);
msg.param.remove(Param::WebxdcSummary);
msg.param.remove(Param::WebxdcSummaryTimestamp);
msg.param.remove(Param::PostMessageFileBytes);
msg.param.remove(Param::PostMessageViewtype);
if msg.download_state != DownloadState::Done {
// we don't use Message.get_text() here,
// because it may change in future,
// when UI shows this info itself,
// then the additional_text will not be added in get_text anymore.
msg.text += &msg.additional_text;
}
if !msg.original_msg_id.is_unset() {
bail!("message already saved.");
}
let copy_fields = "from_id, to_id, timestamp_rcvd, type, txt,
let copy_fields = "from_id, to_id, timestamp_rcvd, type,
mime_modified, mime_headers, mime_compressed, mime_in_reply_to, subject, msgrmsg";
let row_id = context
.sql
@@ -4345,7 +4447,7 @@ pub(crate) async fn save_copy_in_self_talk(
&format!(
"INSERT INTO msgs ({copy_fields},
timestamp_sent,
chat_id, rfc724_mid, state, timestamp, param, starred)
txt, chat_id, rfc724_mid, state, timestamp, param, starred)
SELECT {copy_fields},
-- Outgoing messages on originating device
-- have timestamp_sent == 0.
@@ -4353,10 +4455,11 @@ pub(crate) async fn save_copy_in_self_talk(
-- so UIs display the same timestamp
-- for saved and original message.
IIF(timestamp_sent == 0, timestamp, timestamp_sent),
?, ?, ?, ?, ?, ?
?, ?, ?, ?, ?, ?, ?
FROM msgs WHERE id=?;"
),
(
msg.text,
dest_chat_id,
dest_rfc724_mid,
if msg.from_id == ContactId::SELF {

4
src/chat/chat_tests.rs
View File

@@ -3116,7 +3116,7 @@ async fn test_broadcast_channel_protected_listid() -> Result<()> {
.await?
.grpid;
let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes(), None).await?;
let parsed = mimeparser::MimeMessage::from_bytes(bob, sent.payload.as_bytes()).await?;
assert_eq!(
parsed.get_mailinglist_header().unwrap(),
format!("My Channel <{}>", alice_list_id)
@@ -3311,7 +3311,7 @@ async fn test_leave_broadcast_multidevice() -> Result<()> {
remove_contact_from_chat(bob0, bob_chat_id, ContactId::SELF).await?;
let leave_msg = bob0.pop_sent_msg().await;
let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes(), None).await?;
let parsed = MimeMessage::from_bytes(bob1, leave_msg.payload().as_bytes()).await?;
assert_eq!(parsed.parts[0].msg, "I left the group.");
let rcvd = bob1.recv_msg(&leave_msg).await;

6
src/config.rs
View File

@@ -438,14 +438,14 @@ pub enum Config {
/// using this still run unmodified code.
TestHooks,
/// Return an error from `receive_imf_inner()` for a fully downloaded message. For tests.
FailOnReceivingFullMsg,
/// Enable composing emails with Header Protection as defined in
/// <https://www.rfc-editor.org/rfc/rfc9788.html> "Header Protection for Cryptographically
/// Protected Email".
#[strum(props(default = "1"))]
StdHeaderProtectionComposing,
/// Return an error from `receive_imf_inner()`. For tests.
SimulateReceiveImfError,
}
impl Config {

7
src/context.rs
View File

@@ -1083,13 +1083,6 @@ impl Context {
.await?
.unwrap_or_default(),
);
res.insert(
"fail_on_receiving_full_msg",
self.sql
.get_raw_config("fail_on_receiving_full_msg")
.await?
.unwrap_or_default(),
);
res.insert(
"std_header_protection_composing",
self.sql

1
src/context/context_tests.rs
View File

@@ -297,6 +297,7 @@ async fn test_get_info_completeness() {
"encrypted_device_token",
"stats_last_update",
"stats_last_old_contact_id",
"simulate_receive_imf_error", // only used in tests
];
let t = TestContext::new().await;
let info = t.get_info().await.unwrap();

493
src/download.rs
View File

@@ -1,27 +1,18 @@
//! # Download large messages manually.
use std::cmp::max;
use std::collections::BTreeMap;
use anyhow::{Result, anyhow, bail, ensure};
use deltachat_derive::{FromSql, ToSql};
use serde::{Deserialize, Serialize};
use crate::config::Config;
use crate::context::Context;
use crate::imap::session::Session;
use crate::message::{Message, MsgId, Viewtype};
use crate::mimeparser::{MimeMessage, Part};
use crate::tools::time;
use crate::{EventType, chatlist_events, stock_str};
use crate::log::warn;
use crate::message::{self, Message, MsgId, rfc724_mid_exists};
use crate::{EventType, chatlist_events};
/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
///
/// For better UX, some messages as add-member, non-delivery-reports (NDN) or read-receipts (MDN)
/// should always be downloaded completely to handle them correctly,
/// also in larger groups and if group and contact avatar are attached.
/// Most of these cases are caught by `MIN_DOWNLOAD_LIMIT`.
pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840;
pub(crate) mod pre_msg_metadata;
/// If a message is downloaded only partially
/// and `delete_server_after` is set to small timeouts (eg. "at once"),
@@ -29,6 +20,15 @@ pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 163840;
/// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
/// From this point onward outgoing messages are considered large
/// and get a Pre-Message, which announces the Post-Message.
// this is only about sending so we can modify it any time.
// current value is a bit less than the minimum auto download setting from the UIs (which is 160 KiB)
pub(crate) const PRE_MSG_ATTACHMENT_SIZE_THRESHOLD: u64 = 140_000;
/// Max size for pre messages. A warning is emitted when this is exceeded.
pub(crate) const PRE_MSG_SIZE_WARNING_THRESHOLD: usize = 150_000;
/// Download state of the message.
#[derive(
Debug,
@@ -64,20 +64,8 @@ pub enum DownloadState {
InProgress = 1000,
}
impl Context {
// Returns validated download limit or `None` for "no limit".
pub(crate) async fn download_limit(&self) -> Result<Option<u32>> {
let download_limit = self.get_config_int(Config::DownloadLimit).await?;
if download_limit <= 0 {
Ok(None)
} else {
Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
}
}
}
impl MsgId {
/// Schedules full message download for partially downloaded message.
/// Schedules Post-Message download for partially downloaded message.
pub async fn download_full(self, context: &Context) -> Result<()> {
let msg = Message::load_from_db(context, self).await?;
match msg.download_state() {
@@ -86,11 +74,17 @@ impl MsgId {
}
DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
DownloadState::Available | DownloadState::Failure => {
if msg.rfc724_mid().is_empty() {
return Err(anyhow!("Download not possible, message has no rfc724_mid"));
}
self.update_download_state(context, DownloadState::InProgress)
.await?;
context
.sql
.execute("INSERT INTO download (msg_id) VALUES (?)", (self,))
.execute(
"INSERT INTO download (rfc724_mid, msg_id) VALUES (?,?)",
(msg.rfc724_mid(), msg.id),
)
.await?;
context.scheduler.interrupt_inbox().await;
}
@@ -139,25 +133,14 @@ impl Message {
/// Most messages are downloaded automatically on fetch instead.
pub(crate) async fn download_msg(
context: &Context,
msg_id: MsgId,
rfc724_mid: String,
session: &mut Session,
) -> Result<()> {
let Some(msg) = Message::load_from_db_optional(context, msg_id).await? else {
// If partially downloaded message was already deleted
// we do not know its Message-ID anymore
// so cannot download it.
//
// Probably the message expired due to `delete_device_after`
// setting or was otherwise removed from the device,
// so we don't want it to reappear anyway.
return Ok(());
};
let row = context
.sql
.query_row_optional(
"SELECT uid, folder FROM imap WHERE rfc724_mid=? AND target!=''",
(&msg.rfc724_mid,),
(&rfc724_mid,),
|row| {
let server_uid: u32 = row.get(0)?;
let server_folder: String = row.get(1)?;
@@ -172,7 +155,7 @@ pub(crate) async fn download_msg(
};
session
.fetch_single_msg(context, &server_folder, server_uid, msg.rfc724_mid.clone())
.fetch_single_msg(context, &server_folder, server_uid, rfc724_mid)
.await?;
Ok(())
}
@@ -205,7 +188,7 @@ impl Session {
let mut uid_message_ids: BTreeMap<u32, String> = BTreeMap::new();
uid_message_ids.insert(uid, rfc724_mid);
let (sender, receiver) = async_channel::unbounded();
self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, false, sender)
self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender)
.await?;
if receiver.recv().await.is_err() {
bail!("Failed to fetch UID {uid}");
@@ -214,41 +197,143 @@ impl Session {
}
}
impl MimeMessage {
/// Creates a placeholder part and add that to `parts`.
///
/// To create the placeholder, only the outermost header can be used,
/// the mime-structure itself is not available.
///
/// The placeholder part currently contains a text with size and availability of the message.
pub(crate) async fn create_stub_from_partial_download(
&mut self,
context: &Context,
org_bytes: u32,
) -> Result<()> {
let mut text = format!(
"[{}]",
stock_str::partial_download_msg_body(context, org_bytes).await
);
if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
let until = stock_str::download_availability(
context,
time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER),
)
.await;
text += format!(" [{until}]").as_str();
};
info!(context, "Partial download: {}", text);
self.do_add_single_part(Part {
typ: Viewtype::Text,
msg: text,
..Default::default()
});
Ok(())
async fn set_msg_state_to_failed(context: &Context, rfc724_mid: &str) -> Result<()> {
if let Some(msg_id) = rfc724_mid_exists(context, rfc724_mid).await? {
// Update download state to failure
// so it can be retried.
//
// On success update_download_state() is not needed
// as receive_imf() already
// set the state and emitted the event.
msg_id
.update_download_state(context, DownloadState::Failure)
.await?;
}
Ok(())
}
async fn available_post_msgs_contains_rfc724_mid(
context: &Context,
rfc724_mid: &str,
) -> Result<bool> {
Ok(context
.sql
.query_get_value::<String>(
"SELECT rfc724_mid FROM available_post_msgs WHERE rfc724_mid=?",
(&rfc724_mid,),
)
.await?
.is_some())
}
async fn remove_from_available_post_msgs_table(context: &Context, rfc724_mid: &str) -> Result<()> {
context
.sql
.execute(
"DELETE FROM available_post_msgs WHERE rfc724_mid=?",
(&rfc724_mid,),
)
.await?;
Ok(())
}
async fn remove_from_download_table(context: &Context, rfc724_mid: &str) -> Result<()> {
context
.sql
.execute("DELETE FROM download WHERE rfc724_mid=?", (&rfc724_mid,))
.await?;
Ok(())
}
// this is a dedicated method because it is used in multiple places.
pub(crate) async fn premessage_is_downloaded_for(
context: &Context,
rfc724_mid: &str,
) -> Result<bool> {
Ok(message::rfc724_mid_exists(context, rfc724_mid)
.await?
.is_some())
}
pub(crate) async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
let rfc724_mids = context
.sql
.query_map_vec("SELECT rfc724_mid FROM download", (), |row| {
let rfc724_mid: String = row.get(0)?;
Ok(rfc724_mid)
})
.await?;
for rfc724_mid in &rfc724_mids {
let res = download_msg(context, rfc724_mid.clone(), session).await;
if res.is_ok() {
remove_from_download_table(context, rfc724_mid).await?;
remove_from_available_post_msgs_table(context, rfc724_mid).await?;
}
if let Err(err) = res {
warn!(
context,
"Failed to download message rfc724_mid={rfc724_mid}: {:#}.", err
);
if !premessage_is_downloaded_for(context, rfc724_mid).await? {
// This is probably a classical email that vanished before we could download it
warn!(
context,
"{rfc724_mid} is probably a classical email that vanished before we could download it"
);
remove_from_download_table(context, rfc724_mid).await?;
} else if available_post_msgs_contains_rfc724_mid(context, rfc724_mid).await? {
warn!(
context,
"{rfc724_mid} is in available_post_msgs table but we failed to fetch it,
so set the message to DownloadState::Failure - probably it was deleted on the server in the meantime"
);
set_msg_state_to_failed(context, rfc724_mid).await?;
remove_from_download_table(context, rfc724_mid).await?;
remove_from_available_post_msgs_table(context, rfc724_mid).await?;
} else {
// leave the message in DownloadState::InProgress;
// it will be downloaded once it arrives.
}
}
}
Ok(())
}
/// Download known post messages without pre_message
/// in order to guard against lost pre-messages:
pub(crate) async fn download_known_post_messages_without_pre_message(
context: &Context,
session: &mut Session,
) -> Result<()> {
let rfc724_mids = context
.sql
.query_map_vec("SELECT rfc724_mid FROM available_post_msgs", (), |row| {
let rfc724_mid: String = row.get(0)?;
Ok(rfc724_mid)
})
.await?;
for rfc724_mid in &rfc724_mids {
if !premessage_is_downloaded_for(context, rfc724_mid).await? {
// Download the Post-Message unconditionally,
// because the Pre-Message got lost.
// The message may be in the wrong order,
// but at least we have it at all.
let res = download_msg(context, rfc724_mid.clone(), session).await;
if res.is_ok() {
remove_from_available_post_msgs_table(context, rfc724_mid).await?;
}
if let Err(err) = res {
warn!(
context,
"download_known_post_messages_without_pre_message: Failed to download message rfc724_mid={rfc724_mid}: {:#}.",
err
);
}
}
}
Ok(())
}
#[cfg(test)]
@@ -256,11 +341,8 @@ mod tests {
use num_traits::FromPrimitive;
use super::*;
use crate::chat::{get_chat_msgs, send_msg};
use crate::ephemeral::Timer;
use crate::message::delete_msgs;
use crate::receive_imf::receive_imf_from_inbox;
use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager};
use crate::chat::send_msg;
use crate::test_utils::TestContext;
#[test]
fn test_downloadstate_values() {
@@ -278,29 +360,6 @@ mod tests {
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_download_limit() -> Result<()> {
let t = TestContext::new_alice().await;
assert_eq!(t.download_limit().await?, None);
t.set_config(Config::DownloadLimit, Some("200000")).await?;
assert_eq!(t.download_limit().await?, Some(200000));
t.set_config(Config::DownloadLimit, Some("20000")).await?;
assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
t.set_config(Config::DownloadLimit, None).await?;
assert_eq!(t.download_limit().await?, None);
for val in &["0", "-1", "-100", "", "foo"] {
t.set_config(Config::DownloadLimit, Some(val)).await?;
assert_eq!(t.download_limit().await?, None);
}
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_update_download_state() -> Result<()> {
let t = TestContext::new_alice().await;
@@ -332,230 +391,4 @@ mod tests {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_receive_imf() -> Result<()> {
let t = TestContext::new_alice().await;
let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
From: bob@example.com\n\
To: alice@example.org\n\
Subject: foo\n\
Message-ID: <Mr.12345678901@example.com>\n\
Chat-Version: 1.0\n\
Date: Sun, 22 Mar 2020 22:37:57 +0000\
Content-Type: text/plain";
receive_imf_from_inbox(
&t,
"Mr.12345678901@example.com",
header.as_bytes(),
false,
Some(100000),
)
.await?;
let msg = t.get_last_msg().await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.get_subject(), "foo");
assert!(
msg.get_text()
.contains(&stock_str::partial_download_msg_body(&t, 100000).await)
);
receive_imf_from_inbox(
&t,
"Mr.12345678901@example.com",
format!("{header}\n\n100k text...").as_bytes(),
false,
None,
)
.await?;
let msg = t.get_last_msg().await;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.get_subject(), "foo");
assert_eq!(msg.get_text(), "100k text...");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_download_and_ephemeral() -> Result<()> {
let t = TestContext::new_alice().await;
let chat_id = t
.create_chat_with_contact("bob", "bob@example.org")
.await
.id;
chat_id
.set_ephemeral_timer(&t, Timer::Enabled { duration: 60 })
.await?;
// download message from bob partially, this must not change the ephemeral timer
receive_imf_from_inbox(
&t,
"first@example.org",
b"From: Bob <bob@example.org>\n\
To: Alice <alice@example.org>\n\
Chat-Version: 1.0\n\
Subject: subject\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\
Content-Type: text/plain",
false,
Some(100000),
)
.await?;
assert_eq!(
chat_id.get_ephemeral_timer(&t).await?,
Timer::Enabled { duration: 60 }
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_status_update_expands_to_nothing() -> Result<()> {
let alice = TestContext::new_alice().await;
let bob = TestContext::new_bob().await;
let chat_id = alice.create_chat(&bob).await.id;
let file = alice.get_blobdir().join("minimal.xdc");
tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?;
let mut instance = Message::new(Viewtype::File);
instance.set_file_and_deduplicate(&alice, &file, None, None)?;
let _sent1 = alice.send_msg(chat_id, &mut instance).await;
alice
.send_webxdc_status_update(instance.id, r#"{"payload":7}"#)
.await?;
alice.flush_status_updates().await?;
let sent2 = alice.pop_sent_msg().await;
let sent2_rfc724_mid = sent2.load_from_db().await.rfc724_mid;
// not downloading the status update results in an placeholder
receive_imf_from_inbox(
&bob,
&sent2_rfc724_mid,
sent2.payload().as_bytes(),
false,
Some(sent2.payload().len() as u32),
)
.await?;
let msg = bob.get_last_msg().await;
let chat_id = msg.chat_id;
assert_eq!(
get_chat_msgs(&bob, chat_id).await?.len(),
E2EE_INFO_MSGS + 1
);
assert_eq!(msg.download_state(), DownloadState::Available);
// downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
// (usually status updates are too small for not being downloaded directly)
receive_imf_from_inbox(
&bob,
&sent2_rfc724_mid,
sent2.payload().as_bytes(),
false,
None,
)
.await?;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), E2EE_INFO_MSGS);
assert!(
Message::load_from_db_optional(&bob, msg.id)
.await?
.is_none()
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_mdn_expands_to_nothing() -> Result<()> {
let bob = TestContext::new_bob().await;
let raw = b"Subject: Message opened\n\
Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
Chat-Version: 1.0\n\
Message-ID: <bar@example.org>\n\
To: Alice <alice@example.org>\n\
From: Bob <bob@example.org>\n\
Content-Type: multipart/report; report-type=disposition-notification;\n\t\
boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
\n\
\n\
--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
Content-Type: text/plain; charset=utf-8\n\
\n\
bla\n\
\n\
\n\
--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
Content-Type: message/disposition-notification\n\
\n\
Reporting-UA: Delta Chat 1.88.0\n\
Original-Recipient: rfc822;bob@example.org\n\
Final-Recipient: rfc822;bob@example.org\n\
Original-Message-ID: <foo@example.org>\n\
Disposition: manual-action/MDN-sent-automatically; displayed\n\
\n\
\n\
--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
";
// not downloading the mdn results in an placeholder
receive_imf_from_inbox(&bob, "bar@example.org", raw, false, Some(raw.len() as u32)).await?;
let msg = bob.get_last_msg().await;
let chat_id = msg.chat_id;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1);
assert_eq!(msg.download_state(), DownloadState::Available);
// downloading the mdn afterwards expands to nothing and deletes the placeholder directly
// (usually mdn are too small for not being downloaded directly)
receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None).await?;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0);
assert!(
Message::load_from_db_optional(&bob, msg.id)
.await?
.is_none()
);
Ok(())
}
/// Tests that fully downloading the message
/// works even if the Message-ID already exists
/// in the database assigned to the trash chat.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_download_trashed() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let imf_raw = b"From: Bob <bob@example.org>\n\
To: Alice <alice@example.org>\n\
Chat-Version: 1.0\n\
Subject: subject\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\
Content-Type: text/plain";
// Download message from Bob partially.
let partial_received_msg =
receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, Some(100000))
.await?
.unwrap();
assert_eq!(partial_received_msg.msg_ids.len(), 1);
// Delete the received message.
// Not it is still in the database,
// but in the trash chat.
delete_msgs(alice, &[partial_received_msg.msg_ids[0]]).await?;
// Fully download message after deletion.
let full_received_msg =
receive_imf_from_inbox(alice, "first@example.org", imf_raw, false, None).await?;
// The message does not reappear.
// However, `receive_imf` should not fail.
assert!(full_received_msg.is_none());
Ok(())
}
}

248
src/download/pre_msg_metadata.rs Normal file
View File

@@ -0,0 +1,248 @@
use anyhow::{Context as _, Result};
use num_traits::ToPrimitive;
use serde::{Deserialize, Serialize};
use crate::context::Context;
use crate::log::warn;
use crate::message::Message;
use crate::message::Viewtype;
use crate::param::{Param, Params};
/// Metadata contained in Pre-Message that describes the Post-Message.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PreMsgMetadata {
/// size of the attachment in bytes
pub(crate) size: u64,
/// Real viewtype of message
pub(crate) viewtype: Viewtype,
/// the original file name
pub(crate) filename: String,
/// Dimensions: width and height of image or video
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) dimensions: Option<(i32, i32)>,
/// Duration of audio file or video in milliseconds
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) duration: Option<i32>,
}
impl PreMsgMetadata {
// Returns PreMsgMetadata for messages with files and None for messages without file attachment
pub(crate) async fn from_msg(context: &Context, message: &Message) -> Result<Option<Self>> {
if !message.viewtype.has_file() {
return Ok(None);
}
let size = message
.get_filebytes(context)
.await?
.context("Unexpected: file has no size")?;
let filename = message
.param
.get(Param::Filename)
.unwrap_or_default()
.to_owned();
let dimensions = {
match (
message.param.get_int(Param::Width),
message.param.get_int(Param::Height),
) {
(None, None) => None,
(Some(width), Some(height)) => Some((width, height)),
_ => {
warn!(context, "Message misses either width or height.");
None
}
}
};
let duration = message.param.get_int(Param::Duration);
Ok(Some(Self {
size,
filename,
viewtype: message.viewtype,
dimensions,
duration,
}))
}
pub(crate) fn to_header_value(&self) -> Result<String> {
Ok(serde_json::to_string(&self)?)
}
pub(crate) fn try_from_header_value(value: &str) -> Result<Self> {
Ok(serde_json::from_str(value)?)
}
}
impl Params {
/// Applies data from pre_msg_metadata to Params
pub(crate) fn apply_from_pre_msg_metadata(
&mut self,
pre_msg_metadata: &PreMsgMetadata,
) -> &mut Self {
self.set(Param::PostMessageFileBytes, pre_msg_metadata.size);
if !pre_msg_metadata.filename.is_empty() {
self.set(Param::Filename, &pre_msg_metadata.filename);
}
self.set_i64(
Param::PostMessageViewtype,
pre_msg_metadata.viewtype.to_i64().unwrap_or_default(),
);
if let Some((width, height)) = pre_msg_metadata.dimensions {
self.set(Param::Width, width);
self.set(Param::Height, height);
}
if let Some(duration) = pre_msg_metadata.duration {
self.set(Param::Duration, duration);
}
self
}
}
#[cfg(test)]
mod tests {
use anyhow::Result;
use pretty_assertions::assert_eq;
use crate::{
message::{Message, Viewtype},
test_utils::{TestContextManager, create_test_image},
};
use super::PreMsgMetadata;
/// Build from message with file attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_build_from_file_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let mut file_msg = Message::new(Viewtype::File);
file_msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &file_msg).await?;
assert_eq!(
pre_mesage_metadata,
Some(PreMsgMetadata {
size: 1_000_000,
viewtype: Viewtype::File,
filename: "test.bin".to_string(),
dimensions: None,
duration: None,
})
);
Ok(())
}
/// Build from message with image attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_build_from_image_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let mut image_msg = Message::new(Viewtype::Image);
let (width, height) = (1080, 1920);
let test_img = create_test_image(width, height)?;
image_msg.set_file_from_bytes(alice, "vacation.png", &test_img, None)?;
// this is usually done while sending,
// but we don't send it here, so we need to call it ourself
image_msg.try_calc_and_set_dimensions(alice).await?;
let pre_mesage_metadata = PreMsgMetadata::from_msg(alice, &image_msg).await?;
assert_eq!(
pre_mesage_metadata,
Some(PreMsgMetadata {
size: 1816098,
viewtype: Viewtype::Image,
filename: "vacation.png".to_string(),
dimensions: Some((width as i32, height as i32)),
duration: None,
})
);
Ok(())
}
/// Test that serialisation results in expected format
#[test]
fn test_serialize_to_header() -> Result<()> {
assert_eq!(
PreMsgMetadata {
size: 1_000_000,
viewtype: Viewtype::File,
filename: "test.bin".to_string(),
dimensions: None,
duration: None,
}
.to_header_value()?,
"{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\"}"
);
assert_eq!(
PreMsgMetadata {
size: 5_342_765,
viewtype: Viewtype::Image,
filename: "vacation.png".to_string(),
dimensions: Some((1080, 1920)),
duration: None,
}
.to_header_value()?,
"{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}"
);
assert_eq!(
PreMsgMetadata {
size: 5_000,
viewtype: Viewtype::Audio,
filename: "audio-DD-MM-YY.ogg".to_string(),
dimensions: None,
duration: Some(152_310),
}
.to_header_value()?,
"{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}"
);
Ok(())
}
/// Test that deserialisation from expected format works
/// This test will become important for compatibility between versions in the future
#[test]
fn test_deserialize_from_header() -> Result<()> {
assert_eq!(
serde_json::from_str::<PreMsgMetadata>(
"{\"size\":1000000,\"viewtype\":\"File\",\"filename\":\"test.bin\",\"dimensions\":null,\"duration\":null}"
)?,
PreMsgMetadata {
size: 1_000_000,
viewtype: Viewtype::File,
filename: "test.bin".to_string(),
dimensions: None,
duration: None,
}
);
assert_eq!(
serde_json::from_str::<PreMsgMetadata>(
"{\"size\":5342765,\"viewtype\":\"Image\",\"filename\":\"vacation.png\",\"dimensions\":[1080,1920]}"
)?,
PreMsgMetadata {
size: 5_342_765,
viewtype: Viewtype::Image,
filename: "vacation.png".to_string(),
dimensions: Some((1080, 1920)),
duration: None,
}
);
assert_eq!(
serde_json::from_str::<PreMsgMetadata>(
"{\"size\":5000,\"viewtype\":\"Audio\",\"filename\":\"audio-DD-MM-YY.ogg\",\"duration\":152310}"
)?,
PreMsgMetadata {
size: 5_000,
viewtype: Viewtype::Audio,
filename: "audio-DD-MM-YY.ogg".to_string(),
dimensions: None,
duration: Some(152_310),
}
);
Ok(())
}
}

18
src/headerdef.rs
View File

@@ -102,6 +102,21 @@ pub enum HeaderDef {
/// used to encrypt and decrypt messages.
/// This secret is sent to a new member in the member-addition message.
ChatBroadcastSecret,
/// A message with a large attachment is split into two MIME messages:
/// A pre-message, which contains everything but the attachment,
/// and a Post-Message.
/// The Pre-Message gets a `Chat-Post-Message-Id` header
/// referencing the Post-Message's rfc724_mid.
ChatPostMessageId,
/// Announce Post-Message metadata in a Pre-Message.
/// contains serialized PreMsgMetadata struct
ChatPostMessageMetadata,
/// This message is preceded by a Pre-Message
/// and thus this message can be skipped while fetching messages.
/// This is a cleartext / unproteced header.
ChatIsPostMessage,
/// [Autocrypt](https://autocrypt.org/) header.
Autocrypt,
@@ -147,6 +162,9 @@ pub enum HeaderDef {
impl HeaderDef {
/// Returns the corresponding header string.
///
/// Format is lower-kebab-case for easy comparisons.
/// This method is used in message receiving and testing.
pub fn get_headername(&self) -> &'static str {
self.into()
}

127
src/imap.rs
View File

@@ -67,7 +67,6 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
X-MICROSOFT-ORIGINAL-MESSAGE-ID\
)])";
const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
#[derive(Debug)]
pub(crate) struct Imap {
@@ -615,12 +614,23 @@ impl Imap {
.context("prefetch")?;
let read_cnt = msgs.len();
let download_limit = context.download_limit().await?;
let mut uids_fetch = Vec::<(u32, bool /* partially? */)>::with_capacity(msgs.len() + 1);
let mut uids_fetch = Vec::<u32>::with_capacity(msgs.len() + 1);
let mut available_post_msgs = Vec::<String>::with_capacity(msgs.len());
let mut download_when_normal_starts = Vec::<String>::with_capacity(msgs.len());
let mut uid_message_ids = BTreeMap::new();
let mut largest_uid_skipped = None;
let delete_target = context.get_delete_msgs_target().await?;
let download_limit = {
let download_limit: Option<u32> =
context.get_config_parsed(Config::DownloadLimit).await?;
if download_limit == Some(0) {
None
} else {
download_limit
}
};
// Store the info about IMAP messages in the database.
for (uid, ref fetch_response) in msgs {
let headers = match get_fetch_headers(fetch_response) {
@@ -632,6 +642,9 @@ impl Imap {
};
let message_id = prefetch_get_message_id(&headers);
let size = fetch_response
.size
.context("imap fetch response does not contain size")?;
// Determine the target folder where the message should be moved to.
//
@@ -706,14 +719,23 @@ impl Imap {
)
.await.context("prefetch_should_download")?
{
match download_limit {
Some(download_limit) => uids_fetch.push((
uid,
fetch_response.size.unwrap_or_default() > download_limit,
)),
None => uids_fetch.push((uid, false)),
}
uid_message_ids.insert(uid, message_id);
if headers
.get_header_value(HeaderDef::ChatIsPostMessage)
.is_some()
{
info!(context, "{message_id:?} is a post-message.");
available_post_msgs.push(message_id.clone());
// whether it fits download size limit
if download_limit.is_none_or(|download_limit| size < download_limit) {
download_when_normal_starts.push(message_id.clone());
}
} else {
info!(context, "{message_id:?} is not a post-message.");
uids_fetch.push(uid);
uid_message_ids.insert(uid, message_id);
};
} else {
largest_uid_skipped = Some(uid);
}
@@ -747,29 +769,10 @@ impl Imap {
};
let actually_download_messages_future = async {
let sender = sender;
let mut uids_fetch_in_batch = Vec::with_capacity(max(uids_fetch.len(), 1));
let mut fetch_partially = false;
uids_fetch.push((0, !uids_fetch.last().unwrap_or(&(0, false)).1));
for (uid, fp) in uids_fetch {
if fp != fetch_partially {
session
.fetch_many_msgs(
context,
folder,
uids_fetch_in_batch.split_off(0),
&uid_message_ids,
fetch_partially,
sender.clone(),
)
.await
.context("fetch_many_msgs")?;
fetch_partially = fp;
}
uids_fetch_in_batch.push(uid);
}
anyhow::Ok(())
session
.fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender)
.await
.context("fetch_many_msgs")
};
let (largest_uid_fetched, fetch_res) =
@@ -804,6 +807,30 @@ impl Imap {
chat::mark_old_messages_as_noticed(context, received_msgs).await?;
if fetch_res.is_ok() {
info!(
context,
"available_post_msgs: {}, download_when_normal_starts: {}",
available_post_msgs.len(),
download_when_normal_starts.len()
);
for rfc724_mid in available_post_msgs {
context
.sql
.insert("INSERT INTO available_post_msgs VALUES (?)", (rfc724_mid,))
.await?;
}
for rfc724_mid in download_when_normal_starts {
context
.sql
.insert(
"INSERT INTO download (rfc724_mid, msg_id) VALUES (?,0)",
(rfc724_mid,),
)
.await?;
}
}
// Now fail if fetching failed, so we will
// establish a new session if this one is broken.
fetch_res?;
@@ -1373,7 +1400,6 @@ impl Session {
folder: &str,
request_uids: Vec<u32>,
uid_message_ids: &BTreeMap<u32, String>,
fetch_partially: bool,
received_msgs_channel: Sender<(u32, Option<ReceivedMsg>)>,
) -> Result<()> {
if request_uids.is_empty() {
@@ -1381,25 +1407,10 @@ impl Session {
}
for (request_uids, set) in build_sequence_sets(&request_uids)? {
info!(
context,
"Starting a {} FETCH of message set \"{}\".",
if fetch_partially { "partial" } else { "full" },
set
);
let mut fetch_responses = self
.uid_fetch(
&set,
if fetch_partially {
BODY_PARTIAL
} else {
BODY_FULL
},
)
.await
.with_context(|| {
format!("fetching messages {} from folder \"{}\"", &set, folder)
})?;
info!(context, "Starting a full FETCH of message set \"{}\".", set);
let mut fetch_responses = self.uid_fetch(&set, BODY_FULL).await.with_context(|| {
format!("fetching messages {} from folder \"{}\"", &set, folder)
})?;
// Map from UIDs to unprocessed FETCH results. We put unprocessed FETCH results here
// when we want to process other messages first.
@@ -1456,11 +1467,7 @@ impl Session {
count += 1;
let is_deleted = fetch_response.flags().any(|flag| flag == Flag::Deleted);
let (body, partial) = if fetch_partially {
(fetch_response.header(), fetch_response.size) // `BODY.PEEK[HEADER]` goes to header() ...
} else {
(fetch_response.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
};
let body = fetch_response.body();
if is_deleted {
info!(context, "Not processing deleted msg {}.", request_uid);
@@ -1494,7 +1501,7 @@ impl Session {
context,
"Passing message UID {} to receive_imf().", request_uid
);
let res = receive_imf_inner(context, rfc724_mid, body, is_seen, partial).await;
let res = receive_imf_inner(context, rfc724_mid, body, is_seen).await;
let received_msg = match res {
Err(err) => {
warn!(context, "receive_imf error: {err:#}.");

2
src/imap/session.rs
View File

@@ -17,6 +17,7 @@ use crate::tools;
/// - Chat-Version to check if a message is a chat message
/// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
/// not necessarily sent by Delta Chat.
/// - Chat-Is-Post-Message to skip it in background fetch or when it is too large
const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
MESSAGE-ID \
DATE \
@@ -24,6 +25,7 @@ const PREFETCH_FLAGS: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[HEADER.FIE
FROM \
IN-REPLY-TO REFERENCES \
CHAT-VERSION \
CHAT-IS-POST-MESSAGE \
AUTO-SUBMITTED \
AUTOCRYPT-SETUP-MESSAGE\
)])";

2
src/internals_for_benches.rs
View File

@@ -21,7 +21,7 @@ pub async fn store_self_keypair(context: &Context, keypair: &KeyPair) -> Result<
}
pub async fn parse_and_get_text(context: &Context, imf_raw: &[u8]) -> Result<String> {
let mime_parser = MimeMessage::from_bytes(context, imf_raw, None).await?;
let mime_parser = MimeMessage::from_bytes(context, imf_raw).await?;
Ok(mime_parser.parts.into_iter().next().unwrap().msg)
}

99
src/message.rs
View File

@@ -8,6 +8,9 @@ use std::str;
use anyhow::{Context as _, Result, ensure, format_err};
use deltachat_contact_tools::{VcardContact, parse_vcard};
use deltachat_derive::{FromSql, ToSql};
use humansize::BINARY;
use humansize::format_size;
use num_traits::FromPrimitive;
use serde::{Deserialize, Serialize};
use tokio::{fs, io};
@@ -425,6 +428,10 @@ pub struct Message {
pub(crate) ephemeral_timer: EphemeralTimer,
pub(crate) ephemeral_timestamp: i64,
pub(crate) text: String,
/// Text that is added to the end of Message.text
///
/// Currently used for adding the download information on pre-messages
pub(crate) additional_text: String,
/// Message subject.
///
@@ -483,7 +490,7 @@ impl Message {
!id.is_special(),
"Can not load special message ID {id} from DB"
);
let msg = context
let mut msg = context
.sql
.query_row_optional(
concat!(
@@ -565,6 +572,7 @@ impl Message {
original_msg_id: row.get("original_msg_id")?,
mime_modified: row.get("mime_modified")?,
text,
additional_text: String::new(),
subject: row.get("subject")?,
param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
hidden: row.get("hidden")?,
@@ -579,9 +587,48 @@ impl Message {
.await
.with_context(|| format!("failed to load message {id} from the database"))?;
if let Some(msg) = &mut msg {
msg.additional_text =
Self::get_additional_text(context, msg.download_state, &msg.param).await?;
}
Ok(msg)
}
/// Returns additional text which is appended to the message's text field
/// when it is loaded from the database.
/// Currently this is used to add infomation to pre-messages of what the download will be and how large it is
async fn get_additional_text(
context: &Context,
download_state: DownloadState,
param: &Params,
) -> Result<String> {
if download_state != DownloadState::Done {
let file_size = param
.get(Param::PostMessageFileBytes)
.and_then(|s| s.parse().ok())
.map(|file_size: usize| format_size(file_size, BINARY))
.unwrap_or("?".to_owned());
let viewtype = param
.get_i64(Param::PostMessageViewtype)
.and_then(Viewtype::from_i64)
.unwrap_or(Viewtype::Unknown);
let file_name = param
.get(Param::Filename)
.map(sanitize_filename)
.unwrap_or("?".to_owned());
return match viewtype {
Viewtype::File => Ok(format!(" [{file_name} - {file_size}]")),
_ => {
let translated_viewtype = viewtype.to_locale_string(context).await;
Ok(format!(" [{translated_viewtype} - {file_size}]"))
}
};
}
Ok(String::new())
}
/// Returns the MIME type of an attached file if it exists.
///
/// If the MIME type is not known, the function guesses the MIME type
@@ -764,7 +811,7 @@ impl Message {
/// Returns the text of the message.
pub fn get_text(&self) -> String {
self.text.clone()
self.text.clone() + &self.additional_text
}
/// Returns message subject.
@@ -786,7 +833,17 @@ impl Message {
}
/// Returns the size of the file in bytes, if applicable.
/// If message is a pre-message, then this returns size of the to be downloaded file.
pub async fn get_filebytes(&self, context: &Context) -> Result<Option<u64>> {
// if download state is not downloaded then return value from from params metadata
if self.download_state != DownloadState::Done
&& let Some(file_size) = self
.param
.get(Param::PostMessageFileBytes)
.and_then(|s| s.parse().ok())
{
return Ok(Some(file_size));
}
if let Some(path) = self.param.get_file_path(context)? {
Ok(Some(get_filebytes(context, &path).await.with_context(
|| format!("failed to get {} size in bytes", path.display()),
@@ -796,6 +853,21 @@ impl Message {
}
}
/// If message is a Pre-Message,
/// then this returns the viewtype it will have when it is downloaded.
#[cfg(test)]
pub(crate) fn get_post_message_viewtype(&self) -> Option<Viewtype> {
if self.download_state != DownloadState::Done
&& let Some(viewtype) = self
.param
.get_i64(Param::PostMessageViewtype)
.and_then(Viewtype::from_i64)
{
return Some(viewtype);
}
None
}
/// Returns width of associated image or video file.
pub fn get_width(&self) -> i32 {
self.param.get_int(Param::Width).unwrap_or_default()
@@ -1676,9 +1748,17 @@ pub async fn delete_msgs_ex(
let update_db = |trans: &mut rusqlite::Transaction| {
trans.execute(
"UPDATE imap SET target=? WHERE rfc724_mid=?",
(target, msg.rfc724_mid),
(target, &msg.rfc724_mid),
)?;
trans.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))?;
trans.execute(
"DELETE FROM download WHERE rfc724_mid=?",
(&msg.rfc724_mid,),
)?;
trans.execute(
"DELETE FROM available_post_msgs WHERE rfc724_mid=?",
(&msg.rfc724_mid,),
)?;
Ok(())
};
if let Err(e) = context.sql.transaction(update_db).await {
@@ -1746,7 +1826,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
"SELECT
m.chat_id AS chat_id,
m.state AS state,
m.download_state as download_state,
m.ephemeral_timer AS ephemeral_timer,
m.param AS param,
m.from_id AS from_id,
@@ -1759,7 +1838,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
|row| {
let chat_id: ChatId = row.get("chat_id")?;
let state: MessageState = row.get("state")?;
let download_state: DownloadState = row.get("download_state")?;
let param: Params = row.get::<_, String>("param")?.parse().unwrap_or_default();
let from_id: ContactId = row.get("from_id")?;
let rfc724_mid: String = row.get("rfc724_mid")?;
@@ -1771,7 +1849,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
id,
chat_id,
state,
download_state,
param,
from_id,
rfc724_mid,
@@ -1804,7 +1881,6 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
id,
curr_chat_id,
curr_state,
curr_download_state,
curr_param,
curr_from_id,
curr_rfc724_mid,
@@ -1814,14 +1890,7 @@ pub async fn markseen_msgs(context: &Context, msg_ids: Vec<MsgId>) -> Result<()>
_curr_ephemeral_timer,
) in msgs
{
if curr_download_state != DownloadState::Done {
if curr_state == MessageState::InFresh {
// Don't mark partially downloaded messages as seen or send a read receipt since
// they are not really seen by the user.
update_msg_state(context, id, MessageState::InNoticed).await?;
updated_chat_ids.insert(curr_chat_id);
}
} else if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
update_msg_state(context, id, MessageState::InSeen).await?;
info!(context, "Seen message {}.", id);

106
src/message/message_tests.rs
View File

@@ -326,112 +326,6 @@ async fn test_markseen_msgs() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_markseen_not_downloaded_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
alice.set_config(Config::DownloadLimit, Some("1")).await?;
let bob = &tcm.bob().await;
let bob_chat_id = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
tcm.section("Bob sends a large message to Alice");
let file_bytes = include_bytes!("../../test-data/image/screenshot.png");
let mut msg = Message::new(Viewtype::Image);
msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?;
let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await;
tcm.section("Alice receives a large message from Bob");
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert!(!msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
assert_eq!(msg.state, MessageState::InFresh);
markseen_msgs(alice, vec![msg.id]).await?;
// A not downloaded message can be seen only if it's seen on another device.
assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed);
// Marking the message as seen again is a no op.
markseen_msgs(alice, vec![msg.id]).await?;
assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed);
msg.id
.update_download_state(alice, DownloadState::InProgress)
.await?;
markseen_msgs(alice, vec![msg.id]).await?;
assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed);
msg.id
.update_download_state(alice, DownloadState::Failure)
.await?;
markseen_msgs(alice, vec![msg.id]).await?;
assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed);
msg.id
.update_download_state(alice, DownloadState::Undecipherable)
.await?;
markseen_msgs(alice, vec![msg.id]).await?;
assert_eq!(msg.id.get_state(alice).await?, MessageState::InNoticed);
assert!(
!alice
.sql
.exists("SELECT COUNT(*) FROM smtp_mdns", ())
.await?
);
alice.set_config(Config::DownloadLimit, None).await?;
// Let's assume that Alice and Bob resolved the problem with encryption.
let old_msg = msg;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.chat_id, old_msg.chat_id);
assert_eq!(msg.download_state, DownloadState::Done);
assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
assert!(msg.get_showpadlock());
// The message state mustn't be downgraded to `InFresh`.
assert_eq!(msg.state, MessageState::InNoticed);
markseen_msgs(alice, vec![msg.id]).await?;
let msg = Message::load_from_db(alice, msg.id).await?;
assert_eq!(msg.state, MessageState::InSeen);
assert_eq!(
alice
.sql
.count("SELECT COUNT(*) FROM smtp_mdns", ())
.await?,
1
);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_msg_seen_on_imap_when_downloaded() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
alice.set_config(Config::DownloadLimit, Some("1")).await?;
let bob = &tcm.bob().await;
let bob_chat_id = tcm.send_recv_accept(alice, bob, "hi").await.chat_id;
let file_bytes = include_bytes!("../../test-data/image/screenshot.png");
let mut msg = Message::new(Viewtype::Image);
msg.set_file_from_bytes(bob, "a.jpg", file_bytes, None)?;
let sent_msg = bob.send_msg(bob_chat_id, &mut msg).await;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert_eq!(msg.state, MessageState::InFresh);
alice.set_config(Config::DownloadLimit, None).await?;
let seen = true;
let rcvd_msg = receive_imf(alice, sent_msg.payload().as_bytes(), seen)
.await
.unwrap()
.unwrap();
assert_eq!(rcvd_msg.chat_id, msg.chat_id);
let msg = Message::load_from_db(alice, *rcvd_msg.msg_ids.last().unwrap())
.await
.unwrap();
assert_eq!(msg.download_state, DownloadState::Done);
assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
assert!(msg.get_showpadlock());
assert_eq!(msg.state, MessageState::InSeen);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_get_state() -> Result<()> {
let alice = TestContext::new_alice().await;

103
src/mimefactory.rs
View File

@@ -21,6 +21,7 @@ use crate::constants::{ASM_SUBJECT, BROADCAST_INCOMPATIBILITY_MSG};
use crate::constants::{Chattype, DC_FROM_HANDSHAKE};
use crate::contact::{Contact, ContactId, Origin};
use crate::context::Context;
use crate::download::pre_msg_metadata::PreMsgMetadata;
use crate::e2ee::EncryptHelper;
use crate::ensure_and_debug_assert;
use crate::ephemeral::Timer as EphemeralTimer;
@@ -59,6 +60,15 @@ pub enum Loaded {
},
}
#[derive(Debug, Clone, PartialEq)]
pub enum PreMessageMode {
/// adds the Chat-Is-Post-Message header in unprotected part
PostMessage,
/// adds the Chat-Post-Message-ID header to protected part
/// also adds metadata and explicitly excludes attachment
PreMessage { post_msg_rfc724_mid: String },
}
/// Helper to construct mime messages.
#[derive(Debug, Clone)]
pub struct MimeFactory {
@@ -146,6 +156,9 @@ pub struct MimeFactory {
/// This field is used to sustain the topic id of webxdcs needed for peer channels.
webxdc_topic: Option<TopicId>,
/// This field is used when this is either a pre-message or a Post-Message.
pre_message_mode: Option<PreMessageMode>,
}
/// Result of rendering a message, ready to be submitted to a send job.
@@ -500,6 +513,7 @@ impl MimeFactory {
sync_ids_to_delete: None,
attach_selfavatar,
webxdc_topic,
pre_message_mode: None,
};
Ok(factory)
}
@@ -548,6 +562,7 @@ impl MimeFactory {
sync_ids_to_delete: None,
attach_selfavatar: false,
webxdc_topic: None,
pre_message_mode: None,
};
Ok(res)
@@ -779,7 +794,10 @@ impl MimeFactory {
headers.push(("Date", mail_builder::headers::raw::Raw::new(date).into()));
let rfc724_mid = match &self.loaded {
Loaded::Message { msg, .. } => msg.rfc724_mid.clone(),
Loaded::Message { msg, .. } => match &self.pre_message_mode {
Some(PreMessageMode::PreMessage { .. }) => create_outgoing_rfc724_mid(),
_ => msg.rfc724_mid.clone(),
},
Loaded::Mdn { .. } => create_outgoing_rfc724_mid(),
};
headers.push((
@@ -893,7 +911,7 @@ impl MimeFactory {
));
}
let is_encrypted = self.encryption_pubkeys.is_some();
let is_encrypted = self.will_be_encrypted();
// Add ephemeral timer for non-MDN messages.
// For MDNs it does not matter because they are not visible
@@ -978,6 +996,22 @@ impl MimeFactory {
"MIME-Version",
mail_builder::headers::raw::Raw::new("1.0").into(),
));
if self.pre_message_mode == Some(PreMessageMode::PostMessage) {
unprotected_headers.push((
"Chat-Is-Post-Message",
mail_builder::headers::raw::Raw::new("1").into(),
));
} else if let Some(PreMessageMode::PreMessage {
post_msg_rfc724_mid,
}) = self.pre_message_mode.clone()
{
protected_headers.push((
"Chat-Post-Message-ID",
mail_builder::headers::message_id::MessageId::new(post_msg_rfc724_mid).into(),
));
}
for header @ (original_header_name, _header_value) in &headers {
let header_name = original_header_name.to_lowercase();
if header_name == "message-id" {
@@ -1119,6 +1153,10 @@ impl MimeFactory {
for (addr, key) in &encryption_pubkeys {
let fingerprint = key.dc_fingerprint().hex();
let cmd = msg.param.get_cmd();
if self.pre_message_mode == Some(PreMessageMode::PostMessage) {
continue;
}
let should_do_gossip = cmd == SystemMessage::MemberAddedToGroup
|| cmd == SystemMessage::SecurejoinMessage
|| multiple_recipients && {
@@ -1831,19 +1869,23 @@ impl MimeFactory {
let footer = if is_reaction { "" } else { &self.selfstatus };
let message_text = format!(
"{}{}{}{}{}{}",
fwdhint.unwrap_or_default(),
quoted_text.unwrap_or_default(),
escape_message_footer_marks(final_text),
if !final_text.is_empty() && !footer.is_empty() {
"\r\n\r\n"
} else {
""
},
if !footer.is_empty() { "-- \r\n" } else { "" },
footer
);
let message_text = if self.pre_message_mode == Some(PreMessageMode::PostMessage) {
"".to_string()
} else {
format!(
"{}{}{}{}{}{}",
fwdhint.unwrap_or_default(),
quoted_text.unwrap_or_default(),
escape_message_footer_marks(final_text),
if !final_text.is_empty() && !footer.is_empty() {
"\r\n\r\n"
} else {
""
},
if !footer.is_empty() { "-- \r\n" } else { "" },
footer
)
};
let mut main_part = MimePart::new("text/plain", message_text);
if is_reaction {
@@ -1875,8 +1917,19 @@ impl MimeFactory {
// add attachment part
if msg.viewtype.has_file() {
let file_part = build_body_file(context, &msg).await?;
parts.push(file_part);
if let Some(PreMessageMode::PreMessage { .. }) = self.pre_message_mode {
let Some(metadata) = PreMsgMetadata::from_msg(context, &msg).await? else {
bail!("Failed to generate metadata for pre-message")
};
headers.push((
HeaderDef::ChatPostMessageMetadata.into(),
mail_builder::headers::raw::Raw::new(metadata.to_header_value()?).into(),
));
} else {
let file_part = build_body_file(context, &msg).await?;
parts.push(file_part);
}
}
if let Some(msg_kml_part) = self.get_message_kml_part() {
@@ -1921,6 +1974,8 @@ impl MimeFactory {
}
}
self.attach_selfavatar =
self.attach_selfavatar && self.pre_message_mode != Some(PreMessageMode::PostMessage);
if self.attach_selfavatar {
match context.get_config(Config::Selfavatar).await? {
Some(path) => match build_avatar_file(context, &path).await {
@@ -1990,6 +2045,20 @@ impl MimeFactory {
Ok(message)
}
pub fn will_be_encrypted(&self) -> bool {
self.encryption_pubkeys.is_some()
}
pub fn set_as_post_message(&mut self) {
self.pre_message_mode = Some(PreMessageMode::PostMessage);
}
pub fn set_as_pre_message_for(&mut self, post_message: &RenderedEmail) {
self.pre_message_mode = Some(PreMessageMode::PreMessage {
post_msg_rfc724_mid: post_message.rfc724_mid.clone(),
});
}
}
fn hidden_recipients() -> Address<'static> {

8
src/mimefactory/mimefactory_tests.rs
View File

@@ -559,7 +559,7 @@ async fn test_render_reply() {
"1.0"
);
let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes(), None)
let _mime_msg = MimeMessage::from_bytes(context, rendered_msg.message.as_bytes())
.await
.unwrap();
}
@@ -757,7 +757,7 @@ async fn test_protected_headers_directive() -> Result<()> {
assert!(msg.get_showpadlock());
assert!(sent.payload.contains("\r\nSubject: [...]\r\n"));
let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes(), None).await?;
let mime = MimeMessage::from_bytes(&alice, sent.payload.as_bytes()).await?;
let mut payload = str::from_utf8(&mime.decoded_data)?.splitn(2, "\r\n\r\n");
let part = payload.next().unwrap();
assert_eq!(
@@ -781,7 +781,7 @@ async fn test_hp_outer_headers() -> Result<()> {
.await?;
chat::send_text_msg(t, chat_id, "hi!".to_string()).await?;
let sent_msg = t.pop_sent_msg().await;
let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes(), None).await?;
let msg = MimeMessage::from_bytes(t, sent_msg.payload.as_bytes()).await?;
assert_eq!(msg.header_exists(HeaderDef::HpOuter), std_hp_composing);
for hdr in ["Date", "From", "Message-ID"] {
assert_eq!(
@@ -811,7 +811,7 @@ async fn test_dont_remove_self() -> Result<()> {
.await;
println!("{}", sent.payload);
let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes(), None)
let mime_message = MimeMessage::from_bytes(alice, sent.payload.as_bytes())
.await
.unwrap();
assert!(!mime_message.header_exists(HeaderDef::ChatGroupPastMembers));

109
src/mimeparser.rs
View File

@@ -23,6 +23,7 @@ use crate::contact::ContactId;
use crate::context::Context;
use crate::decrypt::{try_decrypt, validate_detached_signature};
use crate::dehtml::dehtml;
use crate::download::pre_msg_metadata::PreMsgMetadata;
use crate::events::EventType;
use crate::headerdef::{HeaderDef, HeaderDefMap};
use crate::key::{self, DcKey, Fingerprint, SignedPublicKey, load_self_secret_keyring};
@@ -147,6 +148,23 @@ pub(crate) struct MimeMessage {
/// Sender timestamp in secs since epoch. Allowed to be in the future due to unsynchronized
/// clocks, but not too much.
pub(crate) timestamp_sent: i64,
pub(crate) pre_message: Option<PreMessageMode>,
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum PreMessageMode {
/// This is Post-Message
/// it replaces it's Pre-Message attachment if it exists already,
/// and if the Pre-Message does not exist it is treated as normal message
PostMessage,
/// This is a Pre-Message,
/// it adds a message preview for a Post-Message
/// and it is ignored if the Post-Message was downloaded already
PreMessage {
post_msg_rfc724_mid: String,
metadata: Option<PreMsgMetadata>,
},
}
#[derive(Debug, PartialEq)]
@@ -240,12 +258,9 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
impl MimeMessage {
/// Parse a mime message.
///
/// If `partial` is set, it contains the full message size in bytes.
pub(crate) async fn from_bytes(
context: &Context,
body: &[u8],
partial: Option<u32>,
) -> Result<Self> {
/// This method has some side-effects,
/// such as saving blobs and saving found public keys to the database.
pub(crate) async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
let mail = mailparse::parse_mail(body)?;
let timestamp_rcvd = smeared_time(context);
@@ -302,7 +317,7 @@ impl MimeMessage {
);
(part, part.ctype.mimetype.parse::<Mime>()?)
} else {
// If it's a partially fetched message, there are no subparts.
// Not a valid signed message, handle it as plaintext.
(&mail, mimetype)
}
} else {
@@ -352,6 +367,16 @@ impl MimeMessage {
let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into());
let mut pre_message = if mail
.headers
.get_header_value(HeaderDef::ChatIsPostMessage)
.is_some()
{
Some(PreMessageMode::PostMessage)
} else {
None
};
let mail_raw; // Memory location for a possible decrypted message.
let decrypted_msg; // Decrypted signed OpenPGP message.
let secrets: Vec<String> = context
@@ -580,6 +605,36 @@ impl MimeMessage {
signatures.clear();
}
if let (Ok(mail), true) = (mail, is_encrypted)
&& let Some(post_msg_rfc724_mid) =
mail.headers.get_header_value(HeaderDef::ChatPostMessageId)
{
let post_msg_rfc724_mid = parse_message_id(&post_msg_rfc724_mid)?;
let metadata = if let Some(value) = mail
.headers
.get_header_value(HeaderDef::ChatPostMessageMetadata)
{
match PreMsgMetadata::try_from_header_value(&value) {
Ok(metadata) => Some(metadata),
Err(error) => {
error!(
context,
"failed to parse metadata header in pre-message: {error:#?}"
);
None
}
}
} else {
warn!(context, "expected pre-message to have metadata header");
None
};
pre_message = Some(PreMessageMode::PreMessage {
post_msg_rfc724_mid,
metadata,
});
}
let mut parser = MimeMessage {
parts: Vec::new(),
headers,
@@ -615,33 +670,27 @@ impl MimeMessage {
is_bot: None,
timestamp_rcvd,
timestamp_sent,
pre_message,
};
match partial {
Some(org_bytes) => {
parser
.create_stub_from_partial_download(context, org_bytes)
.await?;
match mail {
Ok(mail) => {
parser.parse_mime_recursive(context, mail, false).await?;
}
None => match mail {
Ok(mail) => {
parser.parse_mime_recursive(context, mail, false).await?;
}
Err(err) => {
let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]";
Err(err) => {
let txt = "[This message cannot be decrypted.\n\n• It might already help to simply reply to this message and ask the sender to send the message again.\n\n• If you just re-installed Delta Chat then it is best if you re-setup Delta Chat now and choose \"Add as second device\" or import a backup.]";
let part = Part {
typ: Viewtype::Text,
msg_raw: Some(txt.to_string()),
msg: txt.to_string(),
// Don't change the error prefix for now,
// receive_imf.rs:lookup_chat_by_reply() checks it.
error: Some(format!("Decrypting failed: {err:#}")),
..Default::default()
};
parser.do_add_single_part(part);
}
},
let part = Part {
typ: Viewtype::Text,
msg_raw: Some(txt.to_string()),
msg: txt.to_string(),
// Don't change the error prefix for now,
// receive_imf.rs:lookup_chat_by_reply() checks it.
error: Some(format!("Decrypting failed: {err:#}")),
..Default::default()
};
parser.do_add_single_part(part);
}
};
let is_location_only = parser.location_kml.is_some() && parser.parts.is_empty();

129
src/mimeparser/mimeparser_tests.rs
View File

@@ -25,58 +25,54 @@ impl AvatarAction {
async fn test_mimeparser_fromheader() {
let ctx = TestContext::new_alice().await;
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi", None)
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, None);
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi", None)
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: g@c.de \n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, None);
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: <g@c.de>\n\nhi", None)
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: <g@c.de>\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, None);
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C <g@c.de>\n\nhi", None)
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: Goetz C <g@c.de>\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, Some("Goetz C".to_string()));
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" <g@c.de>\n\nhi", None)
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: \"Goetz C\" <g@c.de>\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, Some("Goetz C".to_string()));
let mimemsg =
MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C <g@c.de>\n\nhi", None)
.await
.unwrap();
let mimemsg = MimeMessage::from_bytes(&ctx, b"From: =?utf-8?q?G=C3=B6tz?= C <g@c.de>\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, Some("Götz C".to_string()));
// although RFC 2047 says, encoded-words shall not appear inside quoted-string,
// this combination is used in the wild eg. by MailMate
let mimemsg = MimeMessage::from_bytes(
&ctx,
b"From: \"=?utf-8?q?G=C3=B6tz?= C\" <g@c.de>\n\nhi",
None,
)
.await
.unwrap();
let mimemsg =
MimeMessage::from_bytes(&ctx, b"From: \"=?utf-8?q?G=C3=B6tz?= C\" <g@c.de>\n\nhi")
.await
.unwrap();
let contact = mimemsg.from;
assert_eq!(contact.addr, "g@c.de");
assert_eq!(contact.display_name, Some("Götz C".to_string()));
@@ -86,7 +82,7 @@ async fn test_mimeparser_fromheader() {
async fn test_mimeparser_crash() {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/issue_523.txt");
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
@@ -98,7 +94,7 @@ async fn test_mimeparser_crash() {
async fn test_get_rfc724_mid_exists() {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/mail_with_message_id.txt");
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
@@ -112,7 +108,7 @@ async fn test_get_rfc724_mid_exists() {
async fn test_get_rfc724_mid_not_exists() {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/issue_523.txt");
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(mimeparser.get_rfc724_mid(), None);
@@ -324,7 +320,7 @@ async fn test_mailparse_0_16_0_panic() {
// There should be an error, but no panic.
assert!(
MimeMessage::from_bytes(&context.ctx, &raw[..], None)
MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.is_err()
);
@@ -341,7 +337,7 @@ async fn test_parse_first_addr() {
test1\n\
";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await;
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await;
assert!(mimeparser.is_err());
}
@@ -356,7 +352,7 @@ async fn test_get_parent_timestamp() {
\n\
Some reply\n\
";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -402,7 +398,7 @@ async fn test_mimeparser_with_context() {
--==break==--\n\
\n";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
@@ -438,26 +434,26 @@ async fn test_mimeparser_with_avatars() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/mail_attach_txt.eml");
let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(mimeparser.user_avatar, None);
assert_eq!(mimeparser.group_avatar, None);
let raw = include_bytes!("../../test-data/message/mail_with_user_avatar.eml");
let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(mimeparser.parts.len(), 1);
assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
assert!(mimeparser.user_avatar.unwrap().is_change());
assert_eq!(mimeparser.group_avatar, None);
let raw = include_bytes!("../../test-data/message/mail_with_user_avatar_deleted.eml");
let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(mimeparser.parts.len(), 1);
assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete));
assert_eq!(mimeparser.group_avatar, None);
let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml");
let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(mimeparser.parts.len(), 1);
assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
assert!(mimeparser.user_avatar.unwrap().is_change());
@@ -467,9 +463,7 @@ async fn test_mimeparser_with_avatars() {
let raw = include_bytes!("../../test-data/message/mail_with_user_and_group_avatars.eml");
let raw = String::from_utf8_lossy(raw).to_string();
let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:");
let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes(), None)
.await
.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, raw.as_bytes()).await.unwrap();
assert_eq!(mimeparser.parts.len(), 1);
assert_eq!(mimeparser.parts[0].typ, Viewtype::Image);
assert_eq!(mimeparser.user_avatar, None);
@@ -485,7 +479,7 @@ async fn test_mimeparser_with_videochat() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/videochat_invitation.eml");
let mimeparser = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(mimeparser.parts.len(), 1);
assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
assert_eq!(mimeparser.parts[0].param.get(Param::WebrtcRoom), None);
@@ -528,7 +522,7 @@ Content-Disposition: attachment; filename=\"message.kml\"\n\
--==break==--\n\
;";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -578,7 +572,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
";
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -659,7 +653,7 @@ Disposition: manual-action/MDN-sent-automatically; displayed\n\
--outer--\n\
";
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -706,7 +700,7 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
";
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -753,7 +747,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
------=_Part_25_46172632.1581201680436--
"#;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -797,7 +791,7 @@ MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
------=_Part_25_46172632.1581201680436--
"#;
let message = MimeMessage::from_bytes(&t, &raw[..], None).await.unwrap();
let message = MimeMessage::from_bytes(&t, &raw[..]).await.unwrap();
assert_eq!(message.parts.len(), 1);
assert_eq!(message.parts[0].typ, Viewtype::File);
@@ -839,7 +833,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu
----11019878869865180--
"#;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(message.get_subject(), Some("example".to_string()));
@@ -903,7 +897,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu
--------------779C1631600DF3DB8C02E53A--"#;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(message.get_subject(), Some("Test subject".to_string()));
@@ -966,7 +960,7 @@ iVBORw0KGgoAAAANSUhEUgAAACAAAAAeCAAAAABoYUP1AAAAAXNSR0IArs4c6QAAAo1JREFUKJFdkdFu
------=_NextPart_000_0003_01D622B3.CA753E60--
"#;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -1064,7 +1058,7 @@ From: alice <alice@example.org>
Reply
"##;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -1096,7 +1090,7 @@ From: alice <alice@example.org>
> Just a quote.
"##;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -1130,7 +1124,7 @@ On 2020-10-25, Bob wrote:
> A quote.
"##;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(message.get_subject(), Some("Re: top posting".to_string()));
@@ -1148,7 +1142,7 @@ On 2020-10-25, Bob wrote:
async fn test_attachment_quote() {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/quote_attach.eml");
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
@@ -1166,7 +1160,7 @@ async fn test_attachment_quote() {
async fn test_quote_div() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/gmx-quote.eml");
let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap();
assert_eq!(mimeparser.parts[0].msg, "YIPPEEEEEE\n\nMulti-line");
assert_eq!(mimeparser.parts[0].param.get(Param::Quote).unwrap(), "Now?");
}
@@ -1176,7 +1170,7 @@ async fn test_allinkl_blockquote() {
// all-inkl.com puts quotes into `<blockquote> </blockquote>`.
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/allinkl-quote.eml");
let mimeparser = MimeMessage::from_bytes(&t, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t, raw).await.unwrap();
assert!(mimeparser.parts[0].msg.starts_with("It's 1.0."));
assert_eq!(
mimeparser.parts[0].param.get(Param::Quote).unwrap(),
@@ -1217,7 +1211,7 @@ async fn test_add_subj_to_multimedia_msg() {
async fn test_mime_modified_plain() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/text_plain_unspecified.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
assert!(!mimeparser.is_mime_modified);
assert_eq!(
mimeparser.parts[0].msg,
@@ -1229,7 +1223,7 @@ async fn test_mime_modified_plain() {
async fn test_mime_modified_alt_plain_html() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/text_alt_plain_html.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
assert!(mimeparser.is_mime_modified);
assert_eq!(
mimeparser.parts[0].msg,
@@ -1241,7 +1235,7 @@ async fn test_mime_modified_alt_plain_html() {
async fn test_mime_modified_alt_plain() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/text_alt_plain.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
assert!(!mimeparser.is_mime_modified);
assert_eq!(
mimeparser.parts[0].msg,
@@ -1256,7 +1250,7 @@ async fn test_mime_modified_alt_plain() {
async fn test_mime_modified_alt_html() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/text_alt_html.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
assert!(mimeparser.is_mime_modified);
assert_eq!(
mimeparser.parts[0].msg,
@@ -1268,7 +1262,7 @@ async fn test_mime_modified_alt_html() {
async fn test_mime_modified_html() {
let t = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/text_html.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw, None).await.unwrap();
let mimeparser = MimeMessage::from_bytes(&t.ctx, raw).await.unwrap();
assert!(mimeparser.is_mime_modified);
assert_eq!(
mimeparser.parts[0].msg,
@@ -1288,7 +1282,7 @@ async fn test_mime_modified_large_plain() -> Result<()> {
assert!(long_txt.len() > DC_DESIRED_TEXT_LEN);
{
let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?;
let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?;
assert!(mimemsg.is_mime_modified);
assert!(
mimemsg.parts[0].msg.matches("just repeated").count()
@@ -1321,7 +1315,7 @@ async fn test_mime_modified_large_plain() -> Result<()> {
t.set_config(Config::Bot, Some("1")).await?;
{
let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref(), None).await?;
let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?;
assert!(!mimemsg.is_mime_modified);
assert_eq!(
format!("{}\n", mimemsg.parts[0].msg),
@@ -1368,7 +1362,7 @@ async fn test_x_microsoft_original_message_id() {
MIME-Version: 1.0\n\
\n\
Does it work with outlook now?\n\
", None)
")
.await
.unwrap();
assert_eq!(
@@ -1418,7 +1412,7 @@ async fn test_extra_imf_headers() -> Result<()> {
"Message-ID:",
"Chat-Forty-Two: 42\r\nForty-Two: 42\r\nMessage-ID:",
);
let msg = MimeMessage::from_bytes(t, payload.as_bytes(), None).await?;
let msg = MimeMessage::from_bytes(t, payload.as_bytes()).await?;
assert!(msg.headers.contains_key("chat-version"));
assert!(!msg.headers.contains_key("chat-forty-two"));
assert_ne!(msg.headers.contains_key("forty-two"), std_hp_composing);
@@ -1582,7 +1576,7 @@ async fn test_ms_exchange_mdn() -> Result<()> {
// 1. Test mimeparser directly
let mdn =
include_bytes!("../../test-data/message/ms_exchange_report_disposition_notification.eml");
let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn, None).await?;
let mimeparser = MimeMessage::from_bytes(&t.ctx, mdn).await?;
assert_eq!(mimeparser.mdn_reports.len(), 1);
assert_eq!(
mimeparser.mdn_reports[0].original_message_id.as_deref(),
@@ -1608,7 +1602,6 @@ async fn test_receive_eml() -> Result<()> {
let mime_message = MimeMessage::from_bytes(
&alice,
include_bytes!("../../test-data/message/attached-eml.eml"),
None,
)
.await?;
@@ -1651,7 +1644,6 @@ Content-Disposition: reaction\n\
\n\
\u{1F44D}"
.as_bytes(),
None,
)
.await?;
@@ -1673,7 +1665,7 @@ async fn test_jpeg_as_application_octet_stream() -> Result<()> {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/jpeg-as-application-octet-stream.eml");
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(msg.parts.len(), 1);
@@ -1691,7 +1683,7 @@ async fn test_schleuder() -> Result<()> {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/schleuder.eml");
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(msg.parts.len(), 2);
@@ -1711,7 +1703,7 @@ async fn test_tlsrpt() -> Result<()> {
let context = TestContext::new_alice().await;
let raw = include_bytes!("../../test-data/message/tlsrpt.eml");
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let msg = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(msg.parts.len(), 1);
@@ -1744,7 +1736,6 @@ async fn test_time_in_future() -> Result<()> {
Content-Type: text/plain; charset=utf-8\n\
\n\
Hi",
None,
)
.await?;
@@ -1806,7 +1797,7 @@ Content-Type: text/plain; charset=utf-8
/help
"#;
let message = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let message = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(message.get_subject(), Some("Some subject".to_string()));
@@ -1847,7 +1838,7 @@ async fn test_take_last_header() {
Hello\n\
";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None)
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..])
.await
.unwrap();
assert_eq!(
@@ -1900,9 +1891,7 @@ It DOES end with a linebreak.\r
\r
This is the epilogue. It is also to be ignored.";
let mimeparser = MimeMessage::from_bytes(&context, &raw[..], None)
.await
.unwrap();
let mimeparser = MimeMessage::from_bytes(&context, &raw[..]).await.unwrap();
assert_eq!(mimeparser.parts.len(), 2);
@@ -1948,7 +1937,7 @@ Message with a correct Message-ID hidden header
--luTiGu6GBoVLCvTkzVtmZmwsmhkNMw--
"#;
let message = MimeMessage::from_bytes(t, &raw[..], None).await.unwrap();
let message = MimeMessage::from_bytes(t, &raw[..]).await.unwrap();
assert_eq!(message.get_rfc724_mid().unwrap(), "foo@example.org");
}
@@ -2126,9 +2115,7 @@ Third alternative.
--boundary--
"#;
let message = MimeMessage::from_bytes(context, &raw[..], None)
.await
.unwrap();
let message = MimeMessage::from_bytes(context, &raw[..]).await.unwrap();
assert_eq!(message.parts.len(), 1);
assert_eq!(message.parts[0].typ, Viewtype::Text);
assert_eq!(message.parts[0].msg, "Third alternative.");

30
src/param.rs
View File

@@ -251,6 +251,13 @@ pub enum Param {
/// For info messages: Contact ID in added or removed to a group.
ContactAddedRemoved = b'5',
/// For (pre-)Message: ViewType of the Post-Message,
/// because pre message is always `Viewtype::Text`.
PostMessageViewtype = b'8',
/// For (pre-)Message: File byte size of Post-Message attachment
PostMessageFileBytes = b'9',
}
/// An object for handling key=value parameter lists.
@@ -441,6 +448,15 @@ impl Params {
}
self
}
/// Merge in parameters from other Params struct,
/// overwriting the keys that are in both
/// with the values from the new Params struct.
pub fn merge_in_from_params(&mut self, new_params: Self) -> &mut Self {
let mut new_params = new_params;
self.inner.append(&mut new_params.inner);
self
}
}
#[cfg(test)]
@@ -503,4 +519,18 @@ mod tests {
assert_eq!(p.get(Param::Height), Some("14"));
Ok(())
}
#[test]
fn test_merge() -> Result<()> {
let mut p = Params::from_str("w=12\na=5\nh=14")?;
let p2 = Params::from_str("L=1\nh=17")?;
assert_eq!(p.len(), 3);
p.merge_in_from_params(p2);
assert_eq!(p.len(), 4);
assert_eq!(p.get(Param::Width), Some("12"));
assert_eq!(p.get(Param::Height), Some("17"));
assert_eq!(p.get(Param::Forwarded), Some("5"));
assert_eq!(p.get(Param::IsEdited), Some("1"));
Ok(())
}
}

70
src/reaction.rs
View File

@@ -392,9 +392,8 @@ mod tests {
use crate::chatlist::Chatlist;
use crate::config::Config;
use crate::contact::{Contact, Origin};
use crate::download::DownloadState;
use crate::message::{MessageState, Viewtype, delete_msgs};
use crate::receive_imf::{receive_imf, receive_imf_from_inbox};
use crate::receive_imf::receive_imf;
use crate::sql::housekeeping;
use crate::test_utils::E2EE_INFO_MSGS;
use crate::test_utils::TestContext;
@@ -924,73 +923,6 @@ Content-Disposition: reaction\n\
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_download_and_reaction() -> Result<()> {
let alice = TestContext::new_alice().await;
let bob = TestContext::new_bob().await;
alice
.create_chat_with_contact("Bob", "bob@example.net")
.await;
let msg_header = "From: Bob <bob@example.net>\n\
To: Alice <alice@example.org>\n\
Chat-Version: 1.0\n\
Subject: subject\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\
Content-Type: text/plain";
let msg_full = format!("{msg_header}\n\n100k text...");
// Alice downloads message from Bob partially.
let alice_received_message = receive_imf_from_inbox(
&alice,
"first@example.org",
msg_header.as_bytes(),
false,
Some(100000),
)
.await?
.unwrap();
let alice_msg_id = *alice_received_message.msg_ids.first().unwrap();
// Bob downloads own message on the other device.
let bob_received_message = receive_imf(&bob, msg_full.as_bytes(), false)
.await?
.unwrap();
let bob_msg_id = *bob_received_message.msg_ids.first().unwrap();
// Bob reacts to own message.
send_reaction(&bob, bob_msg_id, "👍").await.unwrap();
let bob_reaction_msg = bob.pop_sent_msg().await;
// Alice receives a reaction.
alice.recv_msg_hidden(&bob_reaction_msg).await;
let reactions = get_msg_reactions(&alice, alice_msg_id).await?;
assert_eq!(reactions.to_string(), "👍1");
let msg = Message::load_from_db(&alice, alice_msg_id).await?;
assert_eq!(msg.download_state(), DownloadState::Available);
// Alice downloads full message.
receive_imf_from_inbox(
&alice,
"first@example.org",
msg_full.as_bytes(),
false,
None,
)
.await?;
// Check that reaction is still on the message after full download.
let msg = Message::load_from_db(&alice, alice_msg_id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
let reactions = get_msg_reactions(&alice, alice_msg_id).await?;
assert_eq!(reactions.to_string(), "👍1");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_send_reaction_multidevice() -> Result<()> {
let mut tcm = TestContextManager::new();

274
src/receive_imf.rs
View File

@@ -20,16 +20,14 @@ use crate::constants::{self, Blocked, Chattype, DC_CHAT_ID_TRASH, EDITED_PREFIX,
use crate::contact::{self, Contact, ContactId, Origin, mark_contact_id_as_verified};
use crate::context::Context;
use crate::debug_logging::maybe_set_logging_xdc_inner;
use crate::download::DownloadState;
use crate::download::{DownloadState, premessage_is_downloaded_for};
use crate::ephemeral::{Timer as EphemeralTimer, stock_ephemeral_timer_changed};
use crate::events::EventType;
use crate::headerdef::{HeaderDef, HeaderDefMap};
use crate::imap::{GENERATED_PREFIX, markseen_on_imap_table};
use crate::key::{DcKey, Fingerprint};
use crate::key::{self_fingerprint, self_fingerprint_opt};
use crate::log::LogExt;
use crate::log::warn;
use crate::logged_debug_assert;
use crate::log::{LogExt as _, warn};
use crate::message::{
self, Message, MessageState, MessengerMessage, MsgId, Viewtype, rfc724_mid_exists,
};
@@ -47,6 +45,7 @@ use crate::tools::{
self, buf_compress, normalize_text, remove_subject_prefix, validate_broadcast_secret,
};
use crate::{chatlist_events, ensure_and_debug_assert, ensure_and_debug_assert_eq, location};
use crate::{logged_debug_assert, mimeparser};
/// This is the struct that is returned after receiving one email (aka MIME message).
///
@@ -157,24 +156,7 @@ pub async fn receive_imf(
let mail = mailparse::parse_mail(imf_raw).context("can't parse mail")?;
let rfc724_mid = crate::imap::prefetch_get_message_id(&mail.headers)
.unwrap_or_else(crate::imap::create_message_id);
if let Some(download_limit) = context.download_limit().await? {
let download_limit: usize = download_limit.try_into()?;
if imf_raw.len() > download_limit {
let head = std::str::from_utf8(imf_raw)?
.split("\r\n\r\n")
.next()
.context("No empty line in the message")?;
return receive_imf_from_inbox(
context,
&rfc724_mid,
head.as_bytes(),
seen,
Some(imf_raw.len().try_into()?),
)
.await;
}
}
receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await
receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen).await
}
/// Emulates reception of a message from "INBOX".
@@ -186,9 +168,8 @@ pub(crate) async fn receive_imf_from_inbox(
rfc724_mid: &str,
imf_raw: &[u8],
seen: bool,
is_partial_download: Option<u32>,
) -> Result<Option<ReceivedMsg>> {
receive_imf_inner(context, rfc724_mid, imf_raw, seen, is_partial_download).await
receive_imf_inner(context, rfc724_mid, imf_raw, seen).await
}
/// Inserts a tombstone into `msgs` table
@@ -211,7 +192,6 @@ async fn get_to_and_past_contact_ids(
context: &Context,
mime_parser: &MimeMessage,
chat_assignment: &ChatAssignment,
is_partial_download: Option<u32>,
parent_message: &Option<Message>,
incoming_origin: Origin,
) -> Result<(Vec<Option<ContactId>>, Vec<Option<ContactId>>)> {
@@ -254,7 +234,7 @@ async fn get_to_and_past_contact_ids(
ChatAssignment::ExistingChat { chat_id, .. } => Some(*chat_id),
ChatAssignment::MailingListOrBroadcast => None,
ChatAssignment::OneOneChat => {
if is_partial_download.is_none() && !mime_parser.incoming {
if !mime_parser.incoming {
parent_message.as_ref().map(|m| m.chat_id)
} else {
None
@@ -484,15 +464,17 @@ async fn get_to_and_past_contact_ids(
/// downloaded again, sets `chat_id=DC_CHAT_ID_TRASH` and returns `Ok(Some(…))`.
/// If the message is so wrong that we didn't even create a database entry,
/// returns `Ok(None)`.
///
/// If `is_partial_download` is set, it contains the full message size in bytes.
pub(crate) async fn receive_imf_inner(
context: &Context,
rfc724_mid: &str,
imf_raw: &[u8],
seen: bool,
is_partial_download: Option<u32>,
) -> Result<Option<ReceivedMsg>> {
ensure!(
!context
.get_config_bool(Config::SimulateReceiveImfError)
.await?
);
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(
context,
@@ -500,16 +482,8 @@ pub(crate) async fn receive_imf_inner(
String::from_utf8_lossy(imf_raw),
);
}
if is_partial_download.is_none() {
ensure!(
!context
.get_config_bool(Config::FailOnReceivingFullMsg)
.await?
);
}
let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw, is_partial_download).await
{
let mut mime_parser = match MimeMessage::from_bytes(context, imf_raw).await {
Err(err) => {
warn!(context, "receive_imf: can't parse MIME: {err:#}.");
if rfc724_mid.starts_with(GENERATED_PREFIX) {
@@ -542,7 +516,15 @@ pub(crate) async fn receive_imf_inner(
// check, if the mail is already in our database.
// make sure, this check is done eg. before securejoin-processing.
let (replace_msg_id, replace_chat_id);
if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
if mime_parser.pre_message == Some(mimeparser::PreMessageMode::PostMessage) {
// Post-Message just replace the attachment and mofified Params, not the whole message
// This is done in the `handle_post_message` method.
replace_msg_id = None;
replace_chat_id = None;
} else if let Some(old_msg_id) = message::rfc724_mid_exists(context, rfc724_mid).await? {
// This code handles the download of old partial download stub messages
// It will be removed after a transitioning period,
// after we have released a few versions with pre-messages
replace_msg_id = Some(old_msg_id);
replace_chat_id = if let Some(msg) = Message::load_from_db_optional(context, old_msg_id)
.await?
@@ -615,11 +597,7 @@ pub(crate) async fn receive_imf_inner(
&mime_parser.from,
fingerprint,
prevent_rename,
is_partial_download.is_some()
&& mime_parser
.get_header(HeaderDef::ContentType)
.unwrap_or_default()
.starts_with("multipart/encrypted"),
false,
)
.await?
{
@@ -651,22 +629,14 @@ pub(crate) async fn receive_imf_inner(
.await?
.filter(|p| Some(p.id) != replace_msg_id);
let chat_assignment = decide_chat_assignment(
context,
&mime_parser,
&parent_message,
rfc724_mid,
from_id,
&is_partial_download,
)
.await?;
let chat_assignment =
decide_chat_assignment(context, &mime_parser, &parent_message, rfc724_mid, from_id).await?;
info!(context, "Chat assignment is {chat_assignment:?}.");
let (to_ids, past_ids) = get_to_and_past_contact_ids(
context,
&mime_parser,
&chat_assignment,
is_partial_download,
&parent_message,
incoming_origin,
)
@@ -773,7 +743,6 @@ pub(crate) async fn receive_imf_inner(
to_id,
allow_creation,
&mut mime_parser,
is_partial_download,
parent_message,
)
.await?;
@@ -789,7 +758,6 @@ pub(crate) async fn receive_imf_inner(
rfc724_mid_orig,
from_id,
seen,
is_partial_download,
replace_msg_id,
prevent_rename,
chat_id,
@@ -957,9 +925,7 @@ pub(crate) async fn receive_imf_inner(
let delete_server_after = context.get_config_delete_server_after().await?;
if !received_msg.msg_ids.is_empty() {
let target = if received_msg.needs_delete_job
|| (delete_server_after == Some(0) && is_partial_download.is_none())
{
let target = if received_msg.needs_delete_job || delete_server_after == Some(0) {
Some(context.get_delete_msgs_target().await?)
} else {
None
@@ -988,7 +954,7 @@ pub(crate) async fn receive_imf_inner(
}
}
if is_partial_download.is_none() && mime_parser.is_call() {
if mime_parser.is_call() {
context
.handle_call_msg(insert_msg_id, &mime_parser, from_id)
.await?;
@@ -1037,7 +1003,7 @@ pub(crate) async fn receive_imf_inner(
/// * `find_key_contact_by_addr`: if true, we only know the e-mail address
/// of the contact, but not the fingerprint,
/// yet want to assign the message to some key-contact.
/// This can happen during prefetch or when the message is partially downloaded.
/// This can happen during prefetch.
/// If we get it wrong, the message will be placed into the correct
/// chat after downloading.
///
@@ -1131,7 +1097,6 @@ async fn decide_chat_assignment(
parent_message: &Option<Message>,
rfc724_mid: &str,
from_id: ContactId,
is_partial_download: &Option<u32>,
) -> Result<ChatAssignment> {
let should_trash = if !mime_parser.mdn_reports.is_empty() {
info!(context, "Message is an MDN (TRASH).");
@@ -1147,9 +1112,39 @@ async fn decide_chat_assignment(
{
info!(context, "Chat edit/delete/iroh/sync message (TRASH).");
true
} else if is_partial_download.is_none()
&& (mime_parser.is_system_message == SystemMessage::CallAccepted
|| mime_parser.is_system_message == SystemMessage::CallEnded)
} else if let Some(pre_message) = &mime_parser.pre_message {
use crate::mimeparser::PreMessageMode::*;
match pre_message {
PostMessage => {
// if pre message exist, then trash after replacing, otherwise treat as normal message
let pre_message_exists = premessage_is_downloaded_for(context, rfc724_mid).await?;
info!(
context,
"Message is a Post-Message ({}).",
if pre_message_exists {
"pre-message exists already, so trash after replacing attachment"
} else {
"no pre-message -> Keep"
}
);
pre_message_exists
}
PreMessage {
post_msg_rfc724_mid,
..
} => {
// if post message already exists, then trash/ignore
let post_msg_exists =
premessage_is_downloaded_for(context, post_msg_rfc724_mid).await?;
info!(
context,
"Message is a Pre-Message (post_msg_exists:{post_msg_exists})."
);
post_msg_exists
}
}
} else if mime_parser.is_system_message == SystemMessage::CallAccepted
|| mime_parser.is_system_message == SystemMessage::CallEnded
{
info!(context, "Call state changed (TRASH).");
true
@@ -1250,7 +1245,7 @@ async fn decide_chat_assignment(
}
} else if let Some(parent) = &parent_message {
if let Some((chat_id, chat_id_blocked)) =
lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
lookup_chat_by_reply(context, mime_parser, parent).await?
{
// Try to assign to a chat based on In-Reply-To/References.
ChatAssignment::ExistingChat {
@@ -1272,7 +1267,7 @@ async fn decide_chat_assignment(
}
} else if let Some(parent) = &parent_message {
if let Some((chat_id, chat_id_blocked)) =
lookup_chat_by_reply(context, mime_parser, parent, is_partial_download).await?
lookup_chat_by_reply(context, mime_parser, parent).await?
{
// Try to assign to a chat based on In-Reply-To/References.
ChatAssignment::ExistingChat {
@@ -1314,7 +1309,6 @@ async fn do_chat_assignment(
to_id: ContactId,
allow_creation: bool,
mime_parser: &mut MimeMessage,
is_partial_download: Option<u32>,
parent_message: Option<Message>,
) -> Result<(ChatId, Blocked, bool)> {
let is_bot = context.get_config_bool(Config::Bot).await?;
@@ -1365,7 +1359,6 @@ async fn do_chat_assignment(
&& let Some((new_chat_id, new_chat_id_blocked)) = create_group(
context,
mime_parser,
is_partial_download.is_some(),
create_blocked,
from_id,
to_ids,
@@ -1414,7 +1407,6 @@ async fn do_chat_assignment(
to_ids,
allow_creation || test_normal_chat.is_some(),
create_blocked,
is_partial_download.is_some(),
)
.await?
{
@@ -1496,7 +1488,6 @@ async fn do_chat_assignment(
&& let Some((new_chat_id, new_chat_id_blocked)) = create_group(
context,
mime_parser,
is_partial_download.is_some(),
Blocked::Not,
from_id,
to_ids,
@@ -1560,7 +1551,6 @@ async fn do_chat_assignment(
to_ids,
allow_creation,
Blocked::Not,
is_partial_download.is_some(),
)
.await?
{
@@ -1641,7 +1631,6 @@ async fn add_parts(
rfc724_mid: &str,
from_id: ContactId,
seen: bool,
is_partial_download: Option<u32>,
mut replace_msg_id: Option<MsgId>,
prevent_rename: bool,
mut chat_id: ChatId,
@@ -1713,10 +1702,9 @@ async fn add_parts(
.get_rfc724_mid()
.unwrap_or(rfc724_mid.to_string());
// Extract ephemeral timer from the message or use the existing timer if the message is not fully downloaded.
let mut ephemeral_timer = if is_partial_download.is_some() {
chat_id.get_ephemeral_timer(context).await?
} else if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer) {
// Extract ephemeral timer from the message
let mut ephemeral_timer = if let Some(value) = mime_parser.get_header(HeaderDef::EphemeralTimer)
{
match value.parse::<EphemeralTimer>() {
Ok(timer) => timer,
Err(err) => {
@@ -1919,7 +1907,6 @@ async fn add_parts(
let chat_id = if better_msg
.as_ref()
.is_some_and(|better_msg| better_msg.is_empty())
&& is_partial_download.is_none()
{
DC_CHAT_ID_TRASH
} else {
@@ -1968,10 +1955,10 @@ async fn add_parts(
}
handle_edit_delete(context, mime_parser, from_id).await?;
handle_post_message(context, mime_parser, from_id).await?;
if is_partial_download.is_none()
&& (mime_parser.is_system_message == SystemMessage::CallAccepted
|| mime_parser.is_system_message == SystemMessage::CallEnded)
if mime_parser.is_system_message == SystemMessage::CallAccepted
|| mime_parser.is_system_message == SystemMessage::CallEnded
{
if let Some(field) = mime_parser.get_header(HeaderDef::InReplyTo) {
if let Some(call) =
@@ -2052,6 +2039,14 @@ async fn add_parts(
}
};
if let Some(mimeparser::PreMessageMode::PreMessage {
metadata: Some(metadata),
..
}) = &mime_parser.pre_message
{
param.apply_from_pre_msg_metadata(metadata);
};
// If you change which information is skipped if the message is trashed,
// also change `MsgId::trash()` and `delete_expired_messages()`
let trash = chat_id.is_trash() || (is_location_kml && part_is_empty && !save_mime_modified);
@@ -2095,14 +2090,20 @@ RETURNING id
"#)?;
let row_id: MsgId = stmt.query_row(params![
replace_msg_id,
rfc724_mid_orig,
if let Some(mimeparser::PreMessageMode::PreMessage {post_msg_rfc724_mid, .. }) = &mime_parser.pre_message {
post_msg_rfc724_mid
} else { rfc724_mid_orig },
if trash { DC_CHAT_ID_TRASH } else { chat_id },
if trash { ContactId::UNDEFINED } else { from_id },
if trash { ContactId::UNDEFINED } else { to_id },
sort_timestamp,
if trash { 0 } else { mime_parser.timestamp_sent },
if trash { 0 } else { mime_parser.timestamp_rcvd },
if trash { Viewtype::Unknown } else { typ },
if trash {
Viewtype::Unknown
} else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message {
Viewtype::Text
} else { typ },
if trash { MessageState::Undefined } else { state },
if trash { MessengerMessage::No } else { is_dc_message },
if trash || hidden { "" } else { msg },
@@ -2114,7 +2115,11 @@ RETURNING id
param.to_string()
},
!trash && hidden,
if trash { 0 } else { part.bytes as isize },
if trash {
0
} else {
part.bytes as isize
},
if save_mime_modified && !(trash || hidden) {
mime_headers.clone()
} else {
@@ -2128,10 +2133,10 @@ RETURNING id
if trash { 0 } else { ephemeral_timestamp },
if trash {
DownloadState::Done
} else if is_partial_download.is_some() {
DownloadState::Available
} else if mime_parser.decrypting_failed {
DownloadState::Undecipherable
} else if let Some(mimeparser::PreMessageMode::PreMessage {..}) = mime_parser.pre_message {
DownloadState::Available
} else {
DownloadState::Done
},
@@ -2324,6 +2329,82 @@ async fn handle_edit_delete(
Ok(())
}
async fn handle_post_message(
context: &Context,
mime_parser: &MimeMessage,
from_id: ContactId,
) -> Result<()> {
if let Some(mimeparser::PreMessageMode::PostMessage) = &mime_parser.pre_message {
// if Pre-Message exist, replace attachment
// only replacing attachment ensures that doesn't overwrite the text if it was edited before.
let rfc724_mid = mime_parser
.get_rfc724_mid()
.context("expected Post-Message to have a message id")?;
let Some(msg_id) = message::rfc724_mid_exists(context, &rfc724_mid).await? else {
warn!(
context,
"Download Post-Message: Database entry does not exist."
);
return Ok(());
};
let Some(original_msg) = Message::load_from_db_optional(context, msg_id).await? else {
// else: message is processed like a normal message
warn!(
context,
"Download Post-Message: pre message was not downloaded, yet so treat as normal message"
);
return Ok(());
};
if original_msg.from_id != from_id {
warn!(context, "Download Post-Message: Bad sender.");
return Ok(());
}
if let Some(part) = mime_parser.parts.first() {
if !part.typ.has_file() {
warn!(
context,
"Download Post-Message: First mime part's message-viewtype has no file"
);
return Ok(());
}
let edit_msg_showpadlock = part
.param
.get_bool(Param::GuaranteeE2ee)
.unwrap_or_default();
if edit_msg_showpadlock || !original_msg.get_showpadlock() {
let mut new_params = original_msg.param.clone();
new_params
.merge_in_from_params(part.param.clone())
.remove(Param::PostMessageFileBytes)
.remove(Param::PostMessageViewtype);
context
.sql
.execute(
"UPDATE msgs SET param=?, type=?, bytes=?, error=?, download_state=? WHERE id=?",
(
new_params.to_string(),
part.typ,
part.bytes as isize,
part.error.as_deref().unwrap_or_default(),
DownloadState::Done as u32,
original_msg.id,
),
)
.await?;
context.emit_msgs_changed(original_msg.chat_id, original_msg.id);
} else {
warn!(context, "Download Post-Message: Not encrypted.");
}
}
}
Ok(())
}
async fn tweak_sort_timestamp(
context: &Context,
mime_parser: &mut MimeMessage,
@@ -2413,7 +2494,6 @@ async fn lookup_chat_by_reply(
context: &Context,
mime_parser: &MimeMessage,
parent: &Message,
is_partial_download: &Option<u32>,
) -> Result<Option<(ChatId, Blocked)>> {
// If the message is encrypted and has group ID,
// lookup by reply should never be needed
@@ -2445,10 +2525,7 @@ async fn lookup_chat_by_reply(
}
// Do not assign unencrypted messages to encrypted chats.
if is_partial_download.is_none()
&& parent_chat.is_encrypted(context).await?
&& !mime_parser.was_encrypted()
{
if parent_chat.is_encrypted(context).await? && !mime_parser.was_encrypted() {
return Ok(None);
}
@@ -2465,18 +2542,7 @@ async fn lookup_or_create_adhoc_group(
to_ids: &[Option<ContactId>],
allow_creation: bool,
create_blocked: Blocked,
is_partial_download: bool,
) -> Result<Option<(ChatId, Blocked, bool)>> {
// Partial download may be an encrypted message with protected Subject header. We do not want to
// create a group with "..." or "Encrypted message" as a subject. The same is for undecipherable
// messages. Instead, assign the message to 1:1 chat with the sender.
if is_partial_download {
info!(
context,
"Ad-hoc group cannot be created from partial download."
);
return Ok(None);
}
if mime_parser.decrypting_failed {
warn!(
context,
@@ -2612,11 +2678,9 @@ async fn is_probably_private_reply(
/// than two members, a new ad hoc group is created.
///
/// On success the function returns the created (chat_id, chat_blocked) tuple.
#[expect(clippy::too_many_arguments)]
async fn create_group(
context: &Context,
mime_parser: &mut MimeMessage,
is_partial_download: bool,
create_blocked: Blocked,
from_id: ContactId,
to_ids: &[Option<ContactId>],
@@ -2698,7 +2762,7 @@ async fn create_group(
if let Some(chat_id) = chat_id {
Ok(Some((chat_id, chat_id_blocked)))
} else if is_partial_download || mime_parser.decrypting_failed {
} else if mime_parser.decrypting_failed {
// It is possible that the message was sent to a valid,
// yet unknown group, which was rejected because
// Chat-Group-Name, which is in the encrypted part, was

273
src/receive_imf/receive_imf_tests.rs
View File

@@ -10,7 +10,6 @@ use crate::chat::{
use crate::chatlist::Chatlist;
use crate::constants::DC_GCL_FOR_FORWARDING;
use crate::contact;
use crate::download::MIN_DOWNLOAD_LIMIT;
use crate::imap::prefetch_should_download;
use crate::imex::{ImexMode, imex};
use crate::securejoin::get_securejoin_qr;
@@ -19,8 +18,6 @@ use crate::test_utils::{
};
use crate::tools::{SystemTime, time};
use rand::distr::SampleString;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_outgoing() -> Result<()> {
let context = TestContext::new_alice().await;
@@ -28,7 +25,7 @@ async fn test_outgoing() -> Result<()> {
From: alice@example.org\n\
\n\
hello";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await?;
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await?;
assert_eq!(mimeparser.incoming, false);
Ok(())
}
@@ -43,7 +40,7 @@ async fn test_bad_from() {
References: <Gr.HcxyMARjyJy.9-uvzWPTLtV@nauta.cu>\n\
\n\
hello\x00";
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..], None).await;
let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).await;
assert!(mimeparser.is_err());
}
@@ -2842,7 +2839,7 @@ References: <second@example.net> <nonexistent@example.net> <first@example.net>
Content-Type: text/plain; charset=utf-8; format=flowed; delsp=no
Message with references."#;
let mime_parser = MimeMessage::from_bytes(&t, &mime[..], None).await?;
let mime_parser = MimeMessage::from_bytes(&t, &mime[..]).await?;
let parent = get_parent_message(&t, &mime_parser).await?.unwrap();
assert_eq!(parent.id, first.id);
@@ -4385,37 +4382,6 @@ async fn test_adhoc_grp_name_no_prefix() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_download_later() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = tcm.alice().await;
alice.set_config(Config::DownloadLimit, Some("1")).await?;
assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
let bob = tcm.bob().await;
let bob_chat = bob.create_chat(&alice).await;
// Generate a random string so OpenPGP does not compress it.
let text =
rand::distr::Alphanumeric.sample_string(&mut rand::rng(), MIN_DOWNLOAD_LIMIT as usize);
let sent_msg = bob.send_text(bob_chat.id, &text).await;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert_eq!(msg.state, MessageState::InFresh);
let hi_msg = tcm.send_recv(&bob, &alice, "hi").await;
alice.set_config(Config::DownloadLimit, None).await?;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Done);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
assert!(msg.timestamp_sort <= hi_msg.timestamp_sort);
Ok(())
}
/// Malice can pretend they have the same address as Alice and sends a message encrypted to Alice's
/// key but signed with another one. Alice must detect that this message is wrongly signed and not
/// treat it as Autocrypt-encrypted.
@@ -4450,162 +4416,6 @@ async fn test_outgoing_msg_forgery() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_group_with_big_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = tcm.alice().await;
let bob = tcm.bob().await;
let ba_contact = bob.add_or_lookup_contact_id(&alice).await;
let ab_chat_id = alice.create_chat(&bob).await.id;
let file_bytes = include_bytes!("../../test-data/image/screenshot.png");
let bob_grp_id = create_group(&bob, "Group").await?;
add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
let mut msg = Message::new(Viewtype::Image);
msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?;
let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
assert!(msg.get_showpadlock());
alice.set_config(Config::DownloadLimit, Some("1")).await?;
assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Available);
let alice_chat = Chat::load_from_db(&alice, msg.chat_id).await?;
// Incomplete message is assigned to 1:1 chat.
assert_eq!(alice_chat.typ, Chattype::Single);
alice.set_config(Config::DownloadLimit, None).await?;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Done);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(msg.viewtype, Viewtype::Image);
assert_ne!(msg.chat_id, alice_chat.id);
let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
assert_eq!(alice_grp.typ, Chattype::Group);
assert_eq!(alice_grp.name, "Group");
assert_eq!(
chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
2
);
// Now Bob can send encrypted messages to Alice.
let bob_grp_id = create_group(&bob, "Group1").await?;
add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
let mut msg = Message::new(Viewtype::Image);
msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)?;
let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
assert!(msg.get_showpadlock());
alice.set_config(Config::DownloadLimit, Some("1")).await?;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Available);
// Until fully downloaded, an encrypted message must sit in the 1:1 chat.
assert_eq!(msg.chat_id, ab_chat_id);
alice.set_config(Config::DownloadLimit, None).await?;
let msg = alice.recv_msg(&sent_msg).await;
assert_eq!(msg.download_state, DownloadState::Done);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(msg.viewtype, Viewtype::Image);
assert_ne!(msg.chat_id, ab_chat_id);
let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
assert_eq!(alice_grp.typ, Chattype::Group);
assert_eq!(alice_grp.name, "Group1");
assert_eq!(
chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
2
);
// The big message must go away from the 1:1 chat.
let msgs = chat::get_chat_msgs(&alice, ab_chat_id).await?;
assert_eq!(msgs.len(), E2EE_INFO_MSGS);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_group_consistency() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = tcm.alice().await;
let bob = tcm.bob().await;
let fiona = tcm.fiona().await;
let bob_id = alice.add_or_lookup_contact_id(&bob).await;
let alice_chat_id = create_group(&alice, "foos").await?;
add_contact_to_chat(&alice, alice_chat_id, bob_id).await?;
send_text_msg(&alice, alice_chat_id, "populate".to_string()).await?;
let add = alice.pop_sent_msg().await;
bob.recv_msg(&add).await;
let bob_chat_id = bob.get_last_msg().await.chat_id;
let contacts = get_chat_contacts(&bob, bob_chat_id).await?;
assert_eq!(contacts.len(), 2);
// Bob receives partial message.
let msg_id = receive_imf_from_inbox(
&bob,
"first@example.org",
b"From: Alice <alice@example.org>\n\
To: <bob@example.net>, <charlie@example.com>\n\
Chat-Version: 1.0\n\
Subject: subject\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\
Content-Type: text/plain
Chat-Group-Member-Added: charlie@example.com",
false,
Some(100000),
)
.await?
.context("no received message")?;
let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?;
// Partial download does not change the member list.
assert_eq!(msg.download_state, DownloadState::Available);
assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts);
// Alice sends normal message to bob, adding fiona.
add_contact_to_chat(
&alice,
alice_chat_id,
alice.add_or_lookup_contact_id(&fiona).await,
)
.await?;
bob.recv_msg(&alice.pop_sent_msg().await).await;
let contacts = get_chat_contacts(&bob, bob_chat_id).await?;
assert_eq!(contacts.len(), 3);
// Bob fully receives the partial message.
let msg_id = receive_imf_from_inbox(
&bob,
"first@example.org",
b"From: Alice <alice@example.org>\n\
To: Bob <bob@example.net>\n\
Chat-Version: 1.0\n\
Subject: subject\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\
Content-Type: text/plain
Chat-Group-Member-Added: charlie@example.com",
false,
None,
)
.await?
.context("no received message")?;
let msg = Message::load_from_db(&bob, msg_id.msg_ids[0]).await?;
// After full download, the old message should not change group state.
assert_eq!(msg.download_state, DownloadState::Done);
assert_eq!(get_chat_contacts(&bob, bob_chat_id).await?, contacts);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_protected_group_add_remove_member_missing_key() -> Result<()> {
let mut tcm = TestContextManager::new();
@@ -4844,48 +4654,6 @@ async fn test_references() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_prefer_references_to_downloaded_msgs() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
bob.set_config(Config::DownloadLimit, Some("1")).await?;
let fiona = &tcm.fiona().await;
let alice_bob_id = tcm.send_recv(bob, alice, "hi").await.from_id;
let alice_fiona_id = tcm.send_recv(fiona, alice, "hi").await.from_id;
let alice_chat_id = create_group(alice, "Group").await?;
add_contact_to_chat(alice, alice_chat_id, alice_bob_id).await?;
// W/o fiona the test doesn't work -- the last message is assigned to the 1:1 chat due to
// `is_probably_private_reply()`.
add_contact_to_chat(alice, alice_chat_id, alice_fiona_id).await?;
let sent = alice.send_text(alice_chat_id, "Hi").await;
let received = bob.recv_msg(&sent).await;
assert_eq!(received.download_state, DownloadState::Done);
let bob_chat_id = received.chat_id;
let file_bytes = include_bytes!("../../test-data/image/screenshot.gif");
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "file", file_bytes, None)?;
let mut sent = alice.send_msg(alice_chat_id, &mut msg).await;
sent.payload = sent
.payload
.replace("References:", "X-Microsoft-Original-References:")
.replace("In-Reply-To:", "X-Microsoft-Original-In-Reply-To:");
let received = bob.recv_msg(&sent).await;
assert_eq!(received.download_state, DownloadState::Available);
assert_ne!(received.chat_id, bob_chat_id);
assert_eq!(received.chat_id, bob.get_chat(alice).await.id);
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "file", file_bytes, None)?;
let sent = alice.send_msg(alice_chat_id, &mut msg).await;
let received = bob.recv_msg(&sent).await;
assert_eq!(received.download_state, DownloadState::Available);
assert_eq!(received.chat_id, bob_chat_id);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_list_from() -> Result<()> {
let t = &TestContext::new_alice().await;
@@ -5363,41 +5131,6 @@ async fn test_outgoing_plaintext_two_member_group() -> Result<()> {
Ok(())
}
/// Tests that large messages are assigned
/// to non-key-contacts if the type is not `multipart/encrypted`.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_download_key_contact_lookup() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
// Create two chats with Alice, both with key-contact and email address contact.
let encrypted_chat = bob.create_chat(alice).await;
let unencrypted_chat = bob.create_email_chat(alice).await;
let seen = false;
let is_partial_download = Some(9999);
let received = receive_imf_from_inbox(
bob,
"3333@example.org",
b"From: alice@example.org\n\
To: bob@example.net\n\
Message-ID: <3333@example.org>\n\
Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
\n\
hello\n",
seen,
is_partial_download,
)
.await?
.unwrap();
assert_ne!(received.chat_id, encrypted_chat.id);
assert_eq!(received.chat_id, unencrypted_chat.id);
Ok(())
}
/// Tests that outgoing unencrypted message
/// is assigned to a chat with email-contact.
///

41
src/scheduler.rs
View File

@@ -14,13 +14,12 @@ pub(crate) use self::connectivity::ConnectivityStore;
use crate::config::{self, Config};
use crate::contact::{ContactId, RecentlySeenLoop};
use crate::context::Context;
use crate::download::{DownloadState, download_msg};
use crate::download::{download_known_post_messages_without_pre_message, download_msgs};
use crate::ephemeral::{self, delete_expired_imap_messages};
use crate::events::EventType;
use crate::imap::{FolderMeaning, Imap, session::Session};
use crate::location;
use crate::log::{LogExt, warn};
use crate::message::MsgId;
use crate::smtp::{Smtp, send_smtp_messages};
use crate::sql;
use crate::stats::maybe_send_stats;
@@ -351,38 +350,6 @@ pub(crate) struct Scheduler {
recently_seen_loop: RecentlySeenLoop,
}
async fn download_msgs(context: &Context, session: &mut Session) -> Result<()> {
let msg_ids = context
.sql
.query_map_vec("SELECT msg_id FROM download", (), |row| {
let msg_id: MsgId = row.get(0)?;
Ok(msg_id)
})
.await?;
for msg_id in msg_ids {
if let Err(err) = download_msg(context, msg_id, session).await {
warn!(context, "Failed to download message {msg_id}: {:#}.", err);
// Update download state to failure
// so it can be retried.
//
// On success update_download_state() is not needed
// as receive_imf() already
// set the state and emitted the event.
msg_id
.update_download_state(context, DownloadState::Failure)
.await?;
}
context
.sql
.execute("DELETE FROM download WHERE msg_id=?", (msg_id,))
.await?;
}
Ok(())
}
async fn inbox_loop(
ctx: Context,
started: oneshot::Sender<()>,
@@ -619,6 +586,11 @@ async fn fetch_idle(
delete_expired_imap_messages(ctx)
.await
.context("delete_expired_imap_messages")?;
//-------
// TODO: verify that this is the correct position for this call
// in order to guard against lost pre-messages:
download_known_post_messages_without_pre_message(ctx, &mut session).await?;
} else if folder_config == Config::ConfiguredInboxFolder {
session.last_full_folder_scan.lock().await.take();
}
@@ -704,6 +676,7 @@ async fn fetch_idle(
Ok(session)
}
/// The simplified IMAP IDLE loop to watch non primary folders (non-inbox folders)
async fn simple_imap_loop(
ctx: Context,
started: oneshot::Sender<()>,

24
src/sql/migrations.rs
View File

@@ -1466,6 +1466,30 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT;
.await?;
}
inc_and_check(&mut migration_version, 144)?;
if dbversion < migration_version {
// `msg_id` in `download` table is not needed anymore,
// but we still keep it so that it's possible to import a backup into an older DC version,
// because we don't always release at the same time on all platforms.
sql.execute_migration(
"CREATE TABLE download_new (
rfc724_mid TEXT NOT NULL DEFAULT '',
msg_id INTEGER NOT NULL DEFAULT 0
) STRICT;
INSERT OR IGNORE INTO download_new (rfc724_mid, msg_id)
SELECT m.rfc724_mid, d.msg_id FROM download d
JOIN msgs m ON d.msg_id = m.id
WHERE m.rfc724_mid IS NOT NULL AND m.rfc724_mid != '';
DROP TABLE download;
ALTER TABLE download_new RENAME TO download;
CREATE TABLE available_post_msgs (
rfc724_mid TEXT NOT NULL
);",
migration_version,
)
.await?;
}
let new_version = sql
.get_raw_config_int(VERSION_CFG)
.await?

49
src/stock_str.rs
View File

@@ -4,7 +4,6 @@ use std::collections::HashMap;
use std::sync::Arc;
use anyhow::{Result, bail};
use humansize::{BINARY, format_size};
use strum::EnumProperty as EnumPropertyTrait;
use strum_macros::EnumProperty;
use tokio::sync::RwLock;
@@ -17,7 +16,6 @@ use crate::contact::{Contact, ContactId};
use crate::context::Context;
use crate::message::{Message, Viewtype};
use crate::param::Param;
use crate::tools::timestamp_to_str;
/// Storage for string translations.
#[derive(Debug, Clone)]
@@ -167,12 +165,6 @@ pub enum StockMessage {
))]
QuotaExceedingMsgBody = 98,
#[strum(props(fallback = "%1$s message"))]
PartialDownloadMsgBody = 99,
#[strum(props(fallback = "Download maximum available until %1$s"))]
DownloadAvailability = 100,
#[strum(props(fallback = "Multi Device Synchronization"))]
SyncMsgSubject = 101,
@@ -423,6 +415,9 @@ https://delta.chat/donate"))]
#[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))]
ChatUnencryptedExplanation = 230,
#[strum(props(fallback = "Contact"))]
Contact = 231,
}
impl StockMessage {
@@ -890,6 +885,11 @@ pub(crate) async fn sticker(context: &Context) -> String {
translated(context, StockMessage::Sticker).await
}
/// Stock string: `Contact`.
pub(crate) async fn contact(context: &Context) -> String {
translated(context, StockMessage::Contact).await
}
/// Stock string: `Device messages`.
pub(crate) async fn device_messages(context: &Context) -> String {
translated(context, StockMessage::DeviceMessages).await
@@ -1119,21 +1119,6 @@ pub(crate) async fn quota_exceeding(context: &Context, highest_usage: u64) -> St
.replace("%%", "%")
}
/// Stock string: `%1$s message` with placeholder replaced by human-readable size.
pub(crate) async fn partial_download_msg_body(context: &Context, org_bytes: u32) -> String {
let size = &format_size(org_bytes, BINARY);
translated(context, StockMessage::PartialDownloadMsgBody)
.await
.replace1(size)
}
/// Stock string: `Download maximum available until %1$s`.
pub(crate) async fn download_availability(context: &Context, timestamp: i64) -> String {
translated(context, StockMessage::DownloadAvailability)
.await
.replace1(&timestamp_to_str(timestamp))
}
/// Stock string: `Incoming Messages`.
pub(crate) async fn incoming_messages(context: &Context) -> String {
translated(context, StockMessage::IncomingMessages).await
@@ -1262,6 +1247,24 @@ pub(crate) async fn chat_unencrypted_explanation(context: &Context) -> String {
translated(context, StockMessage::ChatUnencryptedExplanation).await
}
impl Viewtype {
/// returns Localized name for message viewtype
pub async fn to_locale_string(&self, context: &Context) -> String {
match self {
Viewtype::Image => image(context).await,
Viewtype::Gif => gif(context).await,
Viewtype::Sticker => sticker(context).await,
Viewtype::Audio => audio(context).await,
Viewtype::Voice => voice_message(context).await,
Viewtype::Video => video(context).await,
Viewtype::File => file(context).await,
Viewtype::Webxdc => "Mini App".to_owned(),
Viewtype::Vcard => contact(context).await,
Viewtype::Unknown | Viewtype::Text | Viewtype::Call => self.to_string(),
}
}
}
impl Context {
/// Set the stock string for the [StockMessage].
///

8
src/stock_str/stock_str_tests.rs
View File

@@ -118,14 +118,6 @@ async fn test_quota_exceeding_stock_str() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_partial_download_msg_body() -> Result<()> {
let t = TestContext::new().await;
let str = partial_download_msg_body(&t, 1024 * 1024).await;
assert_eq!(str, "1 MiB message");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_update_device_chats() {
let t = TestContext::new_alice().await;

44
src/test_utils.rs
View File

@@ -10,6 +10,7 @@ use std::path::Path;
use std::sync::{Arc, LazyLock};
use std::time::{Duration, Instant};
use anyhow::Result;
use async_channel::{self as channel, Receiver, Sender};
use chat::ChatItem;
use deltachat_contact_tools::{ContactAddress, EmailAddress};
@@ -711,6 +712,32 @@ impl TestContext {
})
}
pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
self.ctx
.sql
.query_map_vec(
"SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
(msg_id,),
|row| {
let _id: MsgId = row.get(0)?;
let msg_id: MsgId = row.get(1)?;
let mime: String = row.get(2)?;
let recipients: String = row.get(3)?;
Ok((msg_id, mime, recipients))
},
)
.await
.unwrap()
.into_iter()
.map(|(msg_id, mime, recipients)| SentMessage {
payload: mime,
sender_msg_id: msg_id,
sender_context: &self.ctx,
recipients,
})
.collect()
}
/// Retrieves a sent sync message from the db.
///
/// This retrieves and removes a sync message which has been scheduled to send from the jobs
@@ -759,7 +786,7 @@ impl TestContext {
/// unlikely to be affected as the message would be processed again in exactly the
/// same way.
pub(crate) async fn parse_msg(&self, msg: &SentMessage<'_>) -> MimeMessage {
MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes(), None)
MimeMessage::from_bytes(&self.ctx, msg.payload().as_bytes())
.await
.unwrap()
}
@@ -1702,6 +1729,21 @@ Until the false-positive is fixed:
}
}
/// Method to create a test image file
pub(crate) fn create_test_image(width: u32, height: u32) -> Result<Vec<u8>> {
use image::{ImageBuffer, Rgb, RgbImage};
use std::io::Cursor;
let mut img: RgbImage = ImageBuffer::new(width, height);
// fill with some pattern so it stays large after compression
for (x, y, pixel) in img.enumerate_pixels_mut() {
*pixel = Rgb([(x % 255) as u8, (x + y % 255) as u8, (y % 255) as u8]);
}
let mut bytes: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut bytes), image::ImageFormat::Png)?;
Ok(bytes)
}
mod tests {
use super::*;

1
src/tests.rs
View File

@@ -1,3 +1,4 @@
mod account_events;
mod aeap;
mod pre_messages;
mod verified_chats;

6
src/tests/pre_messages.rs Normal file
View File

@@ -0,0 +1,6 @@
mod additional_text;
mod forward_and_save;
mod legacy;
mod receiving;
mod sending;
mod util;

40
src/tests/pre_messages/additional_text.rs Normal file
View File

@@ -0,0 +1,40 @@
use anyhow::Result;
use pretty_assertions::assert_eq;
use crate::message::Viewtype;
use crate::test_utils::TestContextManager;
use crate::tests::pre_messages::util::{
send_large_file_message, send_large_image_message, send_large_webxdc_message,
};
/// Test the addition of the download info to message text
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_additional_text_on_different_viewtypes() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let a_group_id = alice.create_group_with_members("test group", &[bob]).await;
tcm.section("Test metadata preview text for File");
let (pre_message, _, _) =
send_large_file_message(alice, a_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.text, "test".to_owned());
assert_eq!(msg.get_text(), "test [test.bin - 976.56 KiB]".to_owned());
tcm.section("Test metadata preview text for webxdc app");
let (pre_message, _, _) = send_large_webxdc_message(alice, a_group_id).await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.text, "test".to_owned());
assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Webxdc));
assert_eq!(msg.get_text(), "test [Mini App - 976.68 KiB]".to_owned());
tcm.section("Test metadata preview text for Image");
let (pre_message, _, _) = send_large_image_message(alice, a_group_id).await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.text, "test".to_owned());
assert_eq!(msg.get_text(), "test [Image - 146.12 KiB]".to_owned());
Ok(())
}

122
src/tests/pre_messages/forward_and_save.rs Normal file
View File

@@ -0,0 +1,122 @@
//! Tests about forwarding and saving Pre-Messages
use anyhow::Result;
use pretty_assertions::assert_eq;
use crate::chat::{self};
use crate::chat::{forward_msgs, save_msgs};
use crate::chatlist::get_last_message_for_chat;
use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD};
use crate::message::{Message, Viewtype};
use crate::test_utils::TestContextManager;
/// Test that forwarding Pre-Message should forward additional text to not be empty
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_forwarding_pre_message_empty_text() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let pre_message = {
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 2);
smtp_rows.first().expect("Pre-Message exists").to_owned()
};
let bob_msg = bob.recv_msg(&pre_message).await;
assert_eq!(bob_msg.download_state, DownloadState::Available);
bob_msg.chat_id.accept(bob).await?;
tcm.section("forward pre message and check it on bobs side");
forward_msgs(bob, &[bob_msg.id], bob_msg.chat_id).await?;
let forwarded_msg_id = get_last_message_for_chat(bob, bob_msg.chat_id)
.await?
.unwrap();
let forwarded_msg = Message::load_from_db(bob, forwarded_msg_id).await?;
assert_eq!(forwarded_msg.is_forwarded(), true);
assert_eq!(forwarded_msg.download_state(), DownloadState::Done);
assert_eq!(
forwarded_msg
.param
.exists(crate::param::Param::PostMessageFileBytes),
false,
"PostMessageFileBytes not set"
);
assert_eq!(
forwarded_msg
.param
.exists(crate::param::Param::PostMessageViewtype),
false,
"PostMessageViewtype not set"
);
assert_eq!(
forwarded_msg.get_text(),
" [test.bin - 976.56 KiB]".to_owned()
);
assert_eq!(forwarded_msg.get_viewtype(), Viewtype::Text);
assert!(forwarded_msg.additional_text.is_empty());
tcm.section("check it on alices side");
let sent_forward_msg = bob.pop_sent_msg().await;
let alice_forwarded_msg = alice.recv_msg(&sent_forward_msg).await;
assert!(alice_forwarded_msg.additional_text.is_empty());
assert_eq!(alice_forwarded_msg.is_forwarded(), true);
assert_eq!(alice_forwarded_msg.download_state(), DownloadState::Done);
assert_eq!(
alice_forwarded_msg
.param
.exists(crate::param::Param::PostMessageFileBytes),
false,
"PostMessageFileBytes not set"
);
assert_eq!(
alice_forwarded_msg
.param
.exists(crate::param::Param::PostMessageViewtype),
false,
"PostMessageViewtype not set"
);
assert_eq!(
alice_forwarded_msg.get_text(),
" [test.bin - 976.56 KiB]".to_owned()
);
Ok(())
}
/// Test that forwarding Pre-Message should forward additional text to not be empty
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_saving_pre_message_empty_text() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let pre_message = {
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 1_000_000], None)?;
assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 2);
smtp_rows.first().expect("Pre-Message exists").to_owned()
};
let bob_msg = bob.recv_msg(&pre_message).await;
assert_eq!(bob_msg.download_state, DownloadState::Available);
bob_msg.chat_id.accept(bob).await?;
tcm.section("save pre message and check it");
save_msgs(bob, &[bob_msg.id]).await?;
let saved_msg_id = get_last_message_for_chat(bob, bob.get_self_chat().await.id)
.await?
.unwrap();
let saved_msg = Message::load_from_db(bob, saved_msg_id).await?;
assert!(saved_msg.additional_text.is_empty());
assert!(saved_msg.get_original_msg_id(bob).await?.is_some());
assert_eq!(saved_msg.download_state(), DownloadState::Done);
assert_eq!(saved_msg.get_text(), " [test.bin - 976.56 KiB]".to_owned());
Ok(())
}

61
src/tests/pre_messages/legacy.rs Normal file
View File

@@ -0,0 +1,61 @@
//! Test that downloading old stub messages still works
use anyhow::Result;
use crate::download::DownloadState;
use crate::receive_imf::receive_imf_from_inbox;
use crate::test_utils::TestContext;
// The code for downloading stub messages stays
// during the transition perios to pre-messages
// so people can still download their files shortly after they updated.
// After there are a few release with pre-message rolled out,
// we will remove the ability to download stub messages and replace the following test
// so it checks that it doesn't crash or that the messages are replaced by sth.
// like "download failed/expired, please ask sender to send it again"
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_download_stub_message() -> Result<()> {
let t = TestContext::new_alice().await;
let header = "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
From: bob@example.com\n\
To: alice@example.org\n\
Subject: foo\n\
Message-ID: <Mr.12345678901@example.com>\n\
Chat-Version: 1.0\n\
Date: Sun, 22 Mar 2020 22:37:57 +0000\
Content-Type: text/plain";
t.sql
.execute(
r#"INSERT INTO chats VALUES(
11001,100,'bob@example.com',0,'',2,'',
replace('C=1763151754\nt=foo','\n',char(10)),0,0,0,0,0,1763151754,0,NULL,0,'');
"#,
(),
)
.await?;
t.sql.execute(r#"INSERT INTO msgs VALUES(
11001,'Mr.12345678901@example.com','',0,
11001,11001,1,1763151754,10,10,1,0,
'[97.66 KiB message]','','',0,1763151754,1763151754,0,X'',
'','',1,0,'',0,0,0,'foo',10,replace('Hop: From: userid; Date: Mon, 4 Dec 2006 13:51:39 +0000\n\nDKIM Results: Passed=true','\n',char(10)),1,NULL,0);
"#, ()).await?;
let msg = t.get_last_msg().await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.get_subject(), "foo");
assert!(msg.get_text().contains("[97.66 KiB message]"));
receive_imf_from_inbox(
&t,
"Mr.12345678901@example.com",
format!("{header}\n\n100k text...").as_bytes(),
false,
)
.await?;
let msg = t.get_last_msg().await;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.get_subject(), "foo");
assert_eq!(msg.get_text(), "100k text...");
Ok(())
}

522
src/tests/pre_messages/receiving.rs Normal file
View File

@@ -0,0 +1,522 @@
//! Tests about receiving Pre-Messages and Post-Message
use anyhow::Result;
use pretty_assertions::assert_eq;
use crate::EventType;
use crate::chat;
use crate::contact;
use crate::download::{
DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD, pre_msg_metadata::PreMsgMetadata,
};
use crate::message::{Message, MessageState, Viewtype, delete_msgs, markseen_msgs};
use crate::mimeparser::MimeMessage;
use crate::param::Param;
use crate::reaction::{get_msg_reactions, send_reaction};
use crate::test_utils::TestContextManager;
use crate::tests::pre_messages::util::{
send_large_file_message, send_large_image_message, send_large_webxdc_message,
};
use crate::webxdc::StatusUpdateSerial;
/// Test that mimeparser can correctly detect and parse pre-messages and Post-Messages
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_mimeparser_pre_message_and_post_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, post_message, _alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
let parsed_pre_message = MimeMessage::from_bytes(bob, pre_message.payload.as_bytes()).await?;
let parsed_post_message = MimeMessage::from_bytes(bob, post_message.payload.as_bytes()).await?;
assert_eq!(
parsed_post_message.pre_message,
Some(crate::mimeparser::PreMessageMode::PostMessage)
);
assert_eq!(
parsed_pre_message.pre_message,
Some(crate::mimeparser::PreMessageMode::PreMessage {
post_msg_rfc724_mid: parsed_post_message.get_rfc724_mid().unwrap(),
metadata: Some(PreMsgMetadata {
size: 1_000_000,
viewtype: Viewtype::File,
filename: "test.bin".to_string(),
dimensions: None,
duration: None
})
})
);
Ok(())
}
/// Test receiving pre-messages and creation of the placeholder message with the metadata
/// for file attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_pre_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, _post_message, _alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.viewtype, Viewtype::Text);
assert_eq!(msg.text, "test".to_owned());
// test that metadata is correctly returned by methods
assert_eq!(msg.get_filebytes(bob).await?, Some(1_000_000));
assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::File));
assert_eq!(msg.get_filename(), Some("test.bin".to_owned()));
Ok(())
}
/// Test receiving the Post-Message after receiving the pre-message
/// for file attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_pre_message_and_dl_post_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, post_message, _alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.viewtype, Viewtype::Text);
assert!(msg.param.exists(Param::PostMessageViewtype));
assert!(msg.param.exists(Param::PostMessageFileBytes));
assert_eq!(msg.text, "test".to_owned());
let _ = bob.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(bob, msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.viewtype, Viewtype::File);
assert_eq!(msg.param.exists(Param::PostMessageViewtype), false);
assert_eq!(msg.param.exists(Param::PostMessageFileBytes), false);
assert_eq!(msg.text, "test".to_owned());
Ok(())
}
/// Test out of order receiving. Post-Message is received & downloaded before pre-message.
/// In that case pre-message shall be trashed.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_out_of_order_receiving() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, post_message, _alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
let msg = bob.recv_msg(&post_message).await;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.viewtype, Viewtype::File);
let _ = bob.recv_msg_trash(&pre_message).await;
Ok(())
}
/// Test receiving the Post-Message after receiving an edit after receiving the pre-message
/// for file attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_pre_message_then_edit_and_then_dl_post_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, post_message, alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
chat::send_edit_request(alice, alice_msg_id, "new_text".to_owned()).await?;
let edit_request = alice.pop_sent_msg().await;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.text, "test".to_owned());
let _ = bob.recv_msg_trash(&edit_request).await;
let msg = Message::load_from_db(bob, msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.text, "new_text".to_owned());
let _ = bob.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(bob, msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.viewtype, Viewtype::File);
assert_eq!(msg.text, "new_text".to_owned());
Ok(())
}
/// Process normal message with file attachment (neither post nor pre message)
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_normal_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(
alice,
"test.bin",
&vec![0u8; (PRE_MSG_ATTACHMENT_SIZE_THRESHOLD - 10_000) as usize],
None,
)?;
msg.set_text("test".to_owned());
let msg_id = chat::send_msg(alice, alice_group_id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 1);
let message = smtp_rows.first().expect("message exists");
let msg = bob.recv_msg(message).await;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.viewtype, Viewtype::File);
assert_eq!(msg.text, "test".to_owned());
Ok(())
}
/// Test receiving pre-messages and creation of the placeholder message with the metadata
/// for image attachment
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_pre_message_image() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, _post_message, _alice_msg_id) =
send_large_image_message(alice, alice_group_id).await?;
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.viewtype, Viewtype::Text);
assert_eq!(msg.text, "test".to_owned());
// test that metadata is correctly returned by methods
assert_eq!(msg.get_post_message_viewtype(), Some(Viewtype::Image));
// recoded image dimensions
assert_eq!(msg.get_filebytes(bob).await?, Some(149632));
assert_eq!(msg.get_height(), 1280);
assert_eq!(msg.get_width(), 720);
Ok(())
}
/// Test receiving reaction on pre-message
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_reaction_on_pre_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
let (pre_message, post_message, alice_msg_id) =
send_large_file_message(alice, alice_group_id, Viewtype::File, &vec![0u8; 1_000_000])
.await?;
// Bob receives pre-message
let bob_msg = bob.recv_msg(&pre_message).await;
assert_eq!(bob_msg.download_state(), DownloadState::Available);
// Alice sends reaction to her own message
send_reaction(alice, alice_msg_id, "👍").await?;
// Bob receives the reaction
bob.recv_msg_hidden(&alice.pop_sent_msg().await).await;
// Test if Bob sees reaction
let reactions = get_msg_reactions(bob, bob_msg.id).await?;
assert_eq!(reactions.to_string(), "👍1");
// Bob downloads Post-Message
bob.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(bob, bob_msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
// Test if Bob still sees reaction
let reactions = get_msg_reactions(bob, bob_msg.id).await?;
assert_eq!(reactions.to_string(), "👍1");
Ok(())
}
/// Tests that fully downloading the message
/// works but does not reappear when it was already deleted
/// (as in the Message-ID already exists in the database
/// and is assigned to the trash chat).
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_full_download_after_trashed() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let bob_group_id = bob.create_group_with_members("test group", &[alice]).await;
let (pre_message, post_message, _bob_msg_id) =
send_large_file_message(bob, bob_group_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
// Download message from Bob partially.
let alice_msg = alice.recv_msg(&pre_message).await;
// Delete the received message.
// Note that it remains in the database in the trash chat.
delete_msgs(alice, &[alice_msg.id]).await?;
// Fully download message after deletion.
alice.recv_msg_trash(&post_message).await;
// The message does not reappear.
let msg = Message::load_from_db_optional(bob, alice_msg.id).await?;
assert!(msg.is_none());
Ok(())
}
/// Test that webxdc updates are received for pre-messages
/// and available when the Post-Message is downloaded
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_group_id = alice.create_group_with_members("test group", &[bob]).await;
// Alice sends a larger instance and an update
let (pre_message, post_message, alice_sent_instance_msg_id) =
send_large_webxdc_message(alice, alice_group_id).await?;
alice
.send_webxdc_status_update(
alice_sent_instance_msg_id,
r#"{"payload": 7, "summary":"sum", "document":"doc"}"#,
)
.await?;
alice.flush_status_updates().await?;
let webxdc_update = alice.pop_sent_msg().await;
// Bob does not download instance but already receives update
let bob_instance = bob.recv_msg(&pre_message).await;
assert_eq!(bob_instance.download_state, DownloadState::Available);
bob.recv_msg_trash(&webxdc_update).await;
// Bob downloads instance, updates should be assigned correctly
bob.recv_msg_trash(&post_message).await;
let bob_instance = bob.get_last_msg().await;
assert_eq!(bob_instance.viewtype, Viewtype::Webxdc);
assert_eq!(bob_instance.download_state, DownloadState::Done);
assert_eq!(
bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial::new(0))
.await?,
r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"#
);
let info = bob_instance.get_webxdc_info(bob).await?;
assert_eq!(info.document, "doc");
assert_eq!(info.summary, "sum");
Ok(())
}
/// Test mark seen pre-message
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_markseen_pre_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let bob_chat_id = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
tcm.section("Bob sends a large message to Alice");
let (pre_message, post_message, _bob_msg_id) =
send_large_file_message(bob, bob_chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
tcm.section("Alice receives a pre-message message from Bob");
let msg = alice.recv_msg(&pre_message).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
assert_eq!(msg.state, MessageState::InFresh);
tcm.section("Alice marks the pre-message as read and sends a MDN");
markseen_msgs(alice, vec![msg.id]).await?;
assert_eq!(msg.id.get_state(alice).await?, MessageState::InSeen);
assert_eq!(
alice
.sql
.count("SELECT COUNT(*) FROM smtp_mdns", ())
.await?,
1
);
tcm.section("Alice downloads message");
alice.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(alice, msg.id).await?;
assert_eq!(msg.download_state, DownloadState::Done);
assert!(msg.param.get_bool(Param::WantsMdn).unwrap_or_default());
assert_eq!(
msg.state,
MessageState::InSeen,
"The message state mustn't be downgraded to `InFresh`"
);
Ok(())
}
/// Test that pre-message can start a chat
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_pre_msg_can_start_chat() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
tcm.section("establishing a DM chat between alice and bob");
let bob_alice_dm_chat_id = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
tcm.section("Alice prepares chat");
let chat_id = chat::create_group(alice, "my group").await?;
let contacts = contact::Contact::get_all(alice, 0, None).await?;
let alice_bob_id = contacts.first().expect("contact exists");
chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?;
tcm.section("Alice sends large message to promote/start chat");
let (pre_message, _post_message, _alice_msg_id) =
send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
tcm.section("Bob receives the pre-message message from Alice");
let msg = bob.recv_msg(&pre_message).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert_ne!(msg.chat_id, bob_alice_dm_chat_id);
let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?;
assert_eq!(chat.name, "my group");
Ok(())
}
/// Test that Post-Message can start a chat
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_post_msg_can_start_chat() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
tcm.section("establishing a DM chat between alice and bob");
let bob_alice_dm_chat_id = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
tcm.section("Alice prepares chat");
let chat_id = chat::create_group(alice, "my group").await?;
let contacts = contact::Contact::get_all(alice, 0, None).await?;
let alice_bob_id = contacts.first().expect("contact exists");
chat::add_contact_to_chat(alice, chat_id, *alice_bob_id).await?;
tcm.section("Alice sends large message to promote/start chat");
let (_pre_message, post_message, _bob_msg_id) =
send_large_file_message(alice, chat_id, Viewtype::File, &vec![0u8; 1_000_000]).await?;
tcm.section("Bob receives the pre-message message from Alice");
let msg = bob.recv_msg(&post_message).await;
assert_eq!(msg.download_state, DownloadState::Done);
assert_ne!(msg.chat_id, bob_alice_dm_chat_id);
let chat = chat::Chat::load_from_db(bob, msg.chat_id).await?;
assert_eq!(chat.name, "my group");
Ok(())
}
/// Test that message ordering is still correct after downloading
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_download_later_keeps_message_order() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
tcm.section(
"establishing a DM chat between alice and bob and bob sends large message to alice",
);
let bob_alice_dm_chat = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
let (pre_message, post_message, _bob_msg_id) = send_large_file_message(
bob,
bob_alice_dm_chat,
Viewtype::File,
&vec![0u8; 1_000_000],
)
.await?;
tcm.section("Alice downloads pre-message");
let msg = alice.recv_msg(&pre_message).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id);
tcm.section("Bob sends hi to Alice");
let hi_msg = tcm.send_recv(bob, alice, "hi").await;
assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
tcm.section("Alice downloads Post-Message");
alice.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(alice, msg.id).await?;
assert_eq!(msg.download_state, DownloadState::Done);
assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
assert!(msg.timestamp_sort <= hi_msg.timestamp_sort);
Ok(())
}
/// Test that ChatlistItemChanged event is emitted when downloading Post-Message
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_chatlist_event_on_post_msg_download() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
tcm.section(
"establishing a DM chat between alice and bob and bob sends large message to alice",
);
let bob_alice_dm_chat = bob.create_chat(alice).await.id;
alice.create_chat(bob).await; // Make sure the chat is accepted.
let (pre_message, post_message, _bob_msg_id) = send_large_file_message(
bob,
bob_alice_dm_chat,
Viewtype::File,
&vec![0u8; 1_000_000],
)
.await?;
tcm.section("Alice downloads pre-message");
let msg = alice.recv_msg(&pre_message).await;
assert_eq!(msg.download_state, DownloadState::Available);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, msg.id);
tcm.section("Alice downloads Post-Message and waits for ChatlistItemChanged event ");
alice.evtracker.clear_events();
alice.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(alice, msg.id).await?;
assert_eq!(msg.download_state, DownloadState::Done);
alice
.evtracker
.get_matching(|e| {
e == &EventType::ChatlistItemChanged {
chat_id: Some(msg.chat_id),
}
})
.await;
Ok(())
}

337
src/tests/pre_messages/sending.rs Normal file
View File

@@ -0,0 +1,337 @@
//! Tests about sending pre-messages
//! - When to send a pre-message and post-message instead of a normal message
//! - Test that sent pre- and post-message contain the right Headers
//! and that they are send in the correct order (pre-message is sent first.)
use anyhow::Result;
use mailparse::MailHeaderMap;
use tokio::fs;
use crate::chat::{self, create_group, send_msg};
use crate::config::Config;
use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
use crate::headerdef::{HeaderDef, HeaderDefMap};
use crate::message::{Message, Viewtype};
use crate::test_utils::{self, TestContext, TestContextManager};
/// Tests that Pre-Message is sent for attachment larger than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
/// Also test that Pre-Message is sent first, before the Post-Message
/// And that Autocrypt-gossip and selfavatar never go into Post-Messages
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_sending_pre_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let fiona = &tcm.fiona().await;
let group_id = alice
.create_group_with_members("test group", &[bob, fiona])
.await;
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
msg.set_text("test".to_owned());
// assert that test attachment is bigger than limit
assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
// Pre-Message and Post-Message should be present
// and test that correct headers are present on both messages
assert_eq!(smtp_rows.len(), 2);
let pre_message = smtp_rows.first().expect("first element exists");
let pre_message_parsed = mailparse::parse_mail(pre_message.payload.as_bytes())?;
let post_message = smtp_rows.get(1).expect("second element exists");
let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?;
assert!(
pre_message_parsed
.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_none()
);
assert!(
post_message_parsed
.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_some()
);
assert_eq!(
post_message_parsed
.headers
.get_header_value(HeaderDef::MessageId),
Some(format!("<{}>", msg.rfc724_mid)),
"Post-Message should have the rfc message id of the database message"
);
assert_ne!(
pre_message_parsed
.headers
.get_header_value(HeaderDef::MessageId),
post_message_parsed
.headers
.get_header_value(HeaderDef::MessageId),
"message ids of Pre-Message and Post-Message should be different"
);
let decrypted_post_message = bob.parse_msg(post_message).await;
assert_eq!(decrypted_post_message.decrypting_failed, false);
assert_eq!(
decrypted_post_message.header_exists(HeaderDef::ChatPostMessageId),
false
);
let decrypted_pre_message = bob.parse_msg(pre_message).await;
assert_eq!(
decrypted_pre_message
.get_header(HeaderDef::ChatPostMessageId)
.map(String::from),
post_message_parsed
.headers
.get_header_value(HeaderDef::MessageId)
);
assert!(
pre_message_parsed
.headers
.get_header_value(HeaderDef::ChatPostMessageId)
.is_none(),
"no Chat-Post-Message-ID header in unprotected headers of Pre-Message"
);
Ok(())
}
/// Tests that Pre-Message has autocrypt gossip headers and self avatar
/// and Post-Message doesn't have these headers
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_selfavatar_and_autocrypt_gossip_goto_pre_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let fiona = &tcm.fiona().await;
let group_id = alice
.create_group_with_members("test group", &[bob, fiona])
.await;
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &[0u8; 300_000], None)?;
msg.set_text("test".to_owned());
// assert that test attachment is bigger than limit
assert!(msg.get_filebytes(alice).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
// simulate conditions for sending self avatar
let avatar_src = alice.get_blobdir().join("avatar.png");
fs::write(&avatar_src, test_utils::AVATAR_900x900_BYTES).await?;
alice
.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
.await?;
let msg_id = chat::send_msg(alice, group_id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 2);
let pre_message = smtp_rows.first().expect("first element exists");
let post_message = smtp_rows.get(1).expect("second element exists");
let post_message_parsed = mailparse::parse_mail(post_message.payload.as_bytes())?;
let decrypted_pre_message = bob.parse_msg(pre_message).await;
assert!(
decrypted_pre_message
.get_header(HeaderDef::ChatPostMessageId)
.is_some(),
"tested message is not a pre-message, sending order may be broken"
);
assert_ne!(decrypted_pre_message.gossiped_keys.len(), 0);
assert_ne!(decrypted_pre_message.user_avatar, None);
let decrypted_post_message = bob.parse_msg(post_message).await;
assert!(
post_message_parsed
.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_some(),
"tested message is not a Post-Message, sending order may be broken"
);
assert_eq!(decrypted_post_message.gossiped_keys.len(), 0);
assert_eq!(decrypted_post_message.user_avatar, None);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unecrypted_gets_no_pre_message() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let chat = alice
.create_chat_with_contact("example", "email@example.org")
.await;
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 300_000], None)?;
msg.set_text("test".to_owned());
let msg_id = chat::send_msg(alice, chat.id, &mut msg).await?;
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 1);
let message_bytes = smtp_rows
.first()
.expect("first element exists")
.payload
.as_bytes();
let message = mailparse::parse_mail(message_bytes)?;
assert!(
message
.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_none(),
);
Ok(())
}
/// Tests that no pre message is sent for normal message
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_not_sending_pre_message_no_attachment() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let chat = alice.create_chat(bob).await;
// send normal text message
let mut msg = Message::new(Viewtype::Text);
msg.set_text("test".to_owned());
let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
let msg = smtp_rows.first().expect("first element exists");
let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
assert!(
mail.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_none(),
"no 'Chat-Is-Post-Message'-header should be present"
);
assert!(
mail.headers
.get_first_header(HeaderDef::ChatPostMessageId.get_headername())
.is_none(),
"no 'Chat-Post-Message-ID'-header should be present in clear text headers"
);
let decrypted_message = bob.parse_msg(msg).await;
assert!(
!decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
"no 'Chat-Post-Message-ID'-header should be present"
);
// test that pre message is not send for large large text
let mut msg = Message::new(Viewtype::Text);
let long_text = String::from_utf8(vec![b'a'; 300_000])?;
assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
msg.set_text(long_text);
let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 1, "only one message should be sent");
let msg = smtp_rows.first().expect("first element exists");
let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
assert!(
mail.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_none()
);
assert!(
mail.headers
.get_first_header(HeaderDef::ChatPostMessageId.get_headername())
.is_none(),
"no 'Chat-Post-Message-ID'-header should be present in clear text headers"
);
let decrypted_message = bob.parse_msg(msg).await;
assert!(
!decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
"no 'Chat-Post-Message-ID'-header should be present"
);
Ok(())
}
/// Tests that no pre message is sent for attachment smaller than `PRE_MSG_ATTACHMENT_SIZE_THRESHOLD`
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_not_sending_pre_message_for_small_attachment() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let chat = alice.create_chat(bob).await;
let mut msg = Message::new(Viewtype::File);
msg.set_file_from_bytes(alice, "test.bin", &vec![0u8; 100_000], None)?;
msg.set_text("test".to_owned());
// assert that test attachment is smaller than limit
assert!(msg.get_filebytes(alice).await?.unwrap() < PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
let msg_id = chat::send_msg(alice, chat.id, &mut msg).await.unwrap();
let smtp_rows = alice.get_smtp_rows_for_msg(msg_id).await;
// only one message and no "is Post-Message" header should be present
assert_eq!(smtp_rows.len(), 1);
let msg = smtp_rows.first().expect("first element exists");
let mail = mailparse::parse_mail(msg.payload.as_bytes())?;
assert!(
mail.headers
.get_first_header(HeaderDef::ChatIsPostMessage.get_headername())
.is_none()
);
assert!(
mail.headers
.get_first_header(HeaderDef::ChatPostMessageId.get_headername())
.is_none(),
"no 'Chat-Post-Message-ID'-header should be present in clear text headers"
);
let decrypted_message = bob.parse_msg(msg).await;
assert!(
!decrypted_message.header_exists(HeaderDef::ChatPostMessageId),
"no 'Chat-Post-Message-ID'-header should be present"
);
Ok(())
}
/// Tests that pre message is not send for large webxdc updates
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_render_webxdc_status_update_object_range() -> Result<()> {
let t = TestContext::new_alice().await;
let chat_id = create_group(&t, "a chat").await?;
let instance = {
let mut instance = Message::new(Viewtype::File);
instance.set_file_from_bytes(
&t,
"minimal.xdc",
include_bytes!("../../../test-data/webxdc/minimal.xdc"),
None,
)?;
let instance_msg_id = send_msg(&t, chat_id, &mut instance).await?;
assert_eq!(instance.viewtype, Viewtype::Webxdc);
Message::load_from_db(&t, instance_msg_id).await
}
.unwrap();
t.pop_sent_msg().await;
assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 0);
let long_text = String::from_utf8(vec![b'a'; 300_000])?;
assert!(long_text.len() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD.try_into().unwrap());
t.send_webxdc_status_update(instance.id, &format!("{{\"payload\": \"{long_text}\"}}"))
.await?;
t.flush_status_updates().await?;
assert_eq!(t.sql.count("SELECT COUNT(*) FROM smtp", ()).await?, 1);
Ok(())
}

65
src/tests/pre_messages/util.rs Normal file
View File

@@ -0,0 +1,65 @@
use anyhow::Result;
use async_zip::tokio::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use futures::io::Cursor as FuturesCursor;
use pretty_assertions::assert_eq;
use tokio_util::compat::FuturesAsyncWriteCompatExt;
use crate::chat::{self, ChatId};
use crate::download::PRE_MSG_ATTACHMENT_SIZE_THRESHOLD;
use crate::message::{Message, MsgId, Viewtype};
use crate::test_utils::{SentMessage, TestContext, create_test_image};
pub async fn send_large_file_message<'a>(
sender: &'a TestContext,
target_chat: ChatId,
view_type: Viewtype,
content: &[u8],
) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
let mut msg = Message::new(view_type);
let file_name = if view_type == Viewtype::Webxdc {
"test.xdc"
} else {
"test.bin"
};
msg.set_file_from_bytes(sender, file_name, content, None)?;
msg.set_text("test".to_owned());
// assert that test attachment is bigger than limit
assert!(msg.get_filebytes(sender).await?.unwrap() > PRE_MSG_ATTACHMENT_SIZE_THRESHOLD);
let msg_id = chat::send_msg(sender, target_chat, &mut msg).await?;
let smtp_rows = sender.get_smtp_rows_for_msg(msg_id).await;
assert_eq!(smtp_rows.len(), 2);
let pre_message = smtp_rows.first().expect("Pre-Message exists");
let post_message = smtp_rows.get(1).expect("Post-Message exists");
Ok((pre_message.to_owned(), post_message.to_owned(), msg_id))
}
pub async fn send_large_webxdc_message<'a>(
sender: &'a TestContext,
target_chat: ChatId,
) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
let futures_cursor = FuturesCursor::new(Vec::new());
let mut buffer = futures_cursor.compat_write();
let mut writer = ZipFileWriter::with_tokio(&mut buffer);
writer
.write_entry_whole(
ZipEntryBuilder::new("index.html".into(), Compression::Stored),
&[0u8; 1_000_000],
)
.await?;
writer.close().await?;
let big_webxdc_app = buffer.into_inner().into_inner();
send_large_file_message(sender, target_chat, Viewtype::Webxdc, &big_webxdc_app).await
}
pub async fn send_large_image_message<'a>(
sender: &'a TestContext,
target_chat: ChatId,
) -> Result<(SentMessage<'a>, SentMessage<'a>, MsgId)> {
let (width, height) = (1080, 1920);
let test_img = create_test_image(width, height)?;
send_large_file_message(sender, target_chat, Viewtype::Image, &test_img).await
}

66
src/webxdc/webxdc_tests.rs
View File

@@ -10,9 +10,8 @@ use crate::chat::{
};
use crate::chatlist::Chatlist;
use crate::config::Config;
use crate::download::DownloadState;
use crate::ephemeral;
use crate::receive_imf::{receive_imf, receive_imf_from_inbox};
use crate::receive_imf::receive_imf;
use crate::test_utils::{E2EE_INFO_MSGS, TestContext, TestContextManager};
use crate::tools::{self, SystemTime};
use crate::{message, sql};
@@ -329,69 +328,6 @@ async fn test_webxdc_contact_request() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
// Alice sends a larger instance and an update
let alice = TestContext::new_alice().await;
let bob = TestContext::new_bob().await;
let chat = alice.create_chat(&bob).await;
bob.set_config(Config::DownloadLimit, Some("40000")).await?;
let mut alice_instance = create_webxdc_instance(
&alice,
"chess.xdc",
include_bytes!("../../test-data/webxdc/chess.xdc"),
)?;
let sent1 = alice.send_msg(chat.id, &mut alice_instance).await;
let alice_instance = sent1.load_from_db().await;
alice
.send_webxdc_status_update(
alice_instance.id,
r#"{"payload": 7, "summary":"sum", "document":"doc"}"#,
)
.await?;
alice.flush_status_updates().await?;
let sent2 = alice.pop_sent_msg().await;
// Bob does not download instance but already receives update
receive_imf_from_inbox(
&bob,
&alice_instance.rfc724_mid,
sent1.payload().as_bytes(),
false,
Some(70790),
)
.await?;
let bob_instance = bob.get_last_msg().await;
bob_instance.chat_id.accept(&bob).await?;
bob.recv_msg_trash(&sent2).await;
assert_eq!(bob_instance.download_state, DownloadState::Available);
// Bob downloads instance, updates should be assigned correctly
let received_msg = receive_imf_from_inbox(
&bob,
&alice_instance.rfc724_mid,
sent1.payload().as_bytes(),
false,
None,
)
.await?
.unwrap();
assert_eq!(*received_msg.msg_ids.first().unwrap(), bob_instance.id);
let bob_instance = bob.get_last_msg().await;
assert_eq!(bob_instance.viewtype, Viewtype::Webxdc);
assert_eq!(bob_instance.download_state, DownloadState::Done);
assert_eq!(
bob.get_webxdc_status_updates(bob_instance.id, StatusUpdateSerial(0))
.await?,
r#"[{"payload":7,"document":"doc","summary":"sum","serial":1,"max_serial":1}]"#
);
let info = bob_instance.get_webxdc_info(&bob).await?;
assert_eq!(info.document, "doc");
assert_eq!(info.summary, "sum");
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_delete_webxdc_instance() -> Result<()> {
let t = TestContext::new_alice().await;