Compare commits

..

3 Commits

Author SHA1 Message Date
Hocuri
0dd4edd4e6 clippy 2026-05-07 12:18:32 +02:00
Hocuri
0df7cc2612 test: Adapt test_stats_key_creation_timestamp() 2026-05-05 23:10:36 +02:00
Hocuri
24876f92aa feat: Reduce resolution of key_create_timestamps in the statistics
Right now, I'm reducing the resolution to 4 weeks (i.e. the timestamp is
rounded to the next multiple of 4 weeks); we could change this.
2026-05-05 23:01:12 +02:00
20 changed files with 137 additions and 144 deletions

9
Cargo.lock generated
View File

@@ -3941,14 +3941,15 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "openssl"
version = "0.10.79"
version = "0.10.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"
checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222"
dependencies = [
"bitflags 2.11.0",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
@@ -3981,9 +3982,9 @@ dependencies = [
[[package]]
name = "openssl-sys"
version = "0.9.115"
version = "0.9.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"
checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6"
dependencies = [
"cc",
"libc",

View File

@@ -4003,6 +4003,8 @@ int dc_msg_get_viewtype (const dc_msg_t* msg);
* Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
*
* Outgoing message states:
* - @ref DC_STATE_OUT_PREPARING - For files which need time to be prepared before they can be sent,
* the message enters this state before @ref DC_STATE_OUT_PENDING. Deprecated.
* - @ref DC_STATE_OUT_DRAFT - Message saved as draft using dc_set_draft()
* - @ref DC_STATE_OUT_PENDING - The user has pressed the "send" button but the
* message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
@@ -5587,6 +5589,13 @@ int64_t dc_lot_get_timestamp (const dc_lot_t* lot);
*/
#define DC_STATE_IN_SEEN 16
/**
* Outgoing message being prepared. See dc_msg_get_state() for details.
*
* @deprecated 2024-12-07
*/
#define DC_STATE_OUT_PREPARING 18
/**
* Outgoing message drafted. See dc_msg_get_state() for details.
*/

View File

@@ -230,6 +230,7 @@ pub enum LotState {
MsgInFresh = 10,
MsgInNoticed = 13,
MsgInSeen = 16,
MsgOutPreparing = 18,
MsgOutDraft = 19,
MsgOutPending = 20,
MsgOutFailed = 24,
@@ -245,6 +246,7 @@ impl From<MessageState> for LotState {
InFresh => LotState::MsgInFresh,
InNoticed => LotState::MsgInNoticed,
InSeen => LotState::MsgInSeen,
OutPreparing => LotState::MsgOutPreparing,
OutDraft => LotState::MsgOutDraft,
OutPending => LotState::MsgOutPending,
OutFailed => LotState::MsgOutFailed,

View File

@@ -190,6 +190,7 @@ class MessageState(IntEnum):
IN_FRESH = 10
IN_NOTICED = 13
IN_SEEN = 16
OUT_PREPARING = 18
OUT_DRAFT = 19
OUT_PENDING = 20
OUT_FAILED = 24

View File

@@ -271,6 +271,15 @@ class Chat:
sent out. This is the same object as was passed in, which
has been modified with the new state of the core.
"""
if msg.is_out_preparing():
assert msg.id != 0
# get a fresh copy of dc_msg, the core needs it
maybe_msg = Message.from_db(self.account, msg.id)
if maybe_msg is not None:
msg = maybe_msg
else:
raise ValueError("message does not exist")
sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
if sent_id == 0:
raise ValueError("message could not be sent")
@@ -324,6 +333,26 @@ class Chat:
raise ValueError("message could not be sent")
return Message.from_db(self.account, sent_id)
def send_prepared(self, message):
"""send a previously prepared message.
:param message: a :class:`Message` instance previously returned by
:meth:`prepare_file`.
:raises ValueError: if message can not be sent.
:returns: a :class:`deltachat.message.Message` instance as sent out.
"""
assert message.id != 0 and message.is_out_preparing()
# get a fresh copy of dc_msg, the core needs it
msg = Message.from_db(self.account, message.id)
# pass 0 as chat-id because core-docs say it's ok when out-preparing
sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
if sent_id == 0:
raise ValueError("message could not be sent")
assert sent_id == msg.id
# modify message in place to avoid bad state for the caller
msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
def set_draft(self, message):
"""set message as draft.

View File

@@ -351,12 +351,17 @@ class Message:
def is_outgoing(self):
"""Return True if Message is outgoing."""
return lib.dc_msg_get_state(self._dc_msg) in (
const.DC_STATE_OUT_PREPARING,
const.DC_STATE_OUT_PENDING,
const.DC_STATE_OUT_FAILED,
const.DC_STATE_OUT_MDN_RCVD,
const.DC_STATE_OUT_DELIVERED,
)
def is_out_preparing(self):
"""Return True if Message is outgoing, but its file is being prepared."""
return self._msgstate == const.DC_STATE_OUT_PREPARING
def is_out_pending(self):
"""Return True if Message is outgoing, but is pending (no single checkmark)."""
return self._msgstate == const.DC_STATE_OUT_PENDING

View File

@@ -2613,7 +2613,7 @@ pub async fn send_msg(context: &Context, chat_id: ChatId, msg: &mut Message) ->
"chat_id cannot be a special chat: {chat_id}"
);
if msg.state != MessageState::Undefined {
if msg.state != MessageState::Undefined && msg.state != MessageState::OutPreparing {
msg.param.remove(Param::GuaranteeE2ee);
msg.param.remove(Param::ForcePlaintext);
// create_send_msg_jobs() will update `param` in the db.
@@ -2721,7 +2721,10 @@ async fn prepare_send_msg(
None
};
if msg.state == MessageState::Undefined
if matches!(
msg.state,
MessageState::Undefined | MessageState::OutPreparing
)
// Legacy SecureJoin "v*-request" messages are unencrypted.
&& msg.param.get_cmd() != SystemMessage::SecurejoinMessage
&& chat.is_encrypted(context).await?
@@ -2934,8 +2937,8 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
UPDATE msgs SET
timestamp=(
SELECT MAX(timestamp) FROM msgs INDEXED BY msgs_index7 WHERE
-- From `InFresh` to `OutDelivered` inclusive, except `OutDraft`.
state IN(10,13,16,18,20,24,26) AND
-- From `InFresh` to `OutMdnRcvd` inclusive except `OutDraft`.
state IN(10,13,16,18,20,24,26,28) AND
hidden IN(0,1) AND
chat_id=? AND
id<=?
@@ -2970,6 +2973,15 @@ WHERE id=?
)?;
for recipients_chunk in recipients.chunks(chunk_size) {
let recipients_chunk = recipients_chunk.join(" ");
if let Some(pre_msg) = &rendered_pre_msg {
let row_id = stmt.execute((
&pre_msg.rfc724_mid,
&recipients_chunk,
&pre_msg.message,
msg.id,
))?;
row_ids.push(row_id.try_into()?);
}
let row_id = stmt.execute((
&rendered_msg.rfc724_mid,
&recipients_chunk,
@@ -2977,16 +2989,6 @@ WHERE id=?
msg.id,
))?;
row_ids.push(row_id.try_into()?);
let Some(pre_msg) = &rendered_pre_msg else {
continue;
};
let row_id = stmt.execute((
&pre_msg.rfc724_mid,
&recipients_chunk,
&pre_msg.message,
msg.id,
))?;
row_ids.push(row_id.try_into()?);
}
Ok(row_ids)
};
@@ -4537,7 +4539,6 @@ pub async fn forward_msgs_2ctx(
msg.state = MessageState::OutPending;
msg.rfc724_mid = create_outgoing_rfc724_mid();
msg.pre_rfc724_mid.clear();
msg.timestamp_sort = curr_timestamp;
chat.prepare_msg_raw(ctx_dst, &mut msg, None).await?;

View File

@@ -1381,8 +1381,13 @@ pub enum MessageState {
/// IMAP and MDN may be sent.
InSeen = 16,
// Deprecated 2024-12-07. Removed 2026-04.
// OutPreparing = 18,
/// For files which need time to be prepared before they can be
/// sent, the message enters this state before
/// OutPending.
///
/// Deprecated 2024-12-07.
OutPreparing = 18,
/// Message saved as draft.
OutDraft = 19,
@@ -1415,6 +1420,7 @@ impl std::fmt::Display for MessageState {
Self::InFresh => "Fresh",
Self::InNoticed => "Noticed",
Self::InSeen => "Seen",
Self::OutPreparing => "Preparing",
Self::OutDraft => "Draft",
Self::OutPending => "Pending",
Self::OutFailed => "Failed",
@@ -1431,7 +1437,7 @@ impl MessageState {
use MessageState::*;
matches!(
self,
OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
)
}
@@ -1440,7 +1446,7 @@ impl MessageState {
use MessageState::*;
matches!(
self,
OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
)
}

View File

@@ -66,6 +66,12 @@ impl Context {
/// Updates `quota.recent`, sets `quota.modified` to the current time
/// and emits an event to let the UIs update connectivity view.
///
/// Moreover, once each time quota gets larger than `QUOTA_WARN_THRESHOLD_PERCENTAGE`,
/// a device message is added.
/// As the message is added only once, the user is not spammed
/// in case for some providers the quota is always at ~100%
/// and new space is allocated as needed.
pub(crate) async fn update_recent_quota(
&self,
session: &mut ImapSession,

View File

@@ -525,10 +525,18 @@ pub(crate) async fn receive_imf_inner(
"Receiving message {rfc724_mid_orig:?}, seen={seen}...",
);
let msg_id = message::rfc724_mid_exists(context, rfc724_mid_orig).await?;
if let Some(msg_id) = msg_id
&& !mime_parser.incoming
{
// These checks must be done before processing of SecureJoin and other special messages.
if mime_parser.pre_message == mimeparser::PreMessageMode::Post {
// Post-Message just replaces the attachment and modifies Params, not the whole message.
// This is done in the `handle_post_message` method.
} else if let Some(msg_id) = message::rfc724_mid_exists(context, rfc724_mid_orig).await? {
info!(
context,
"Message {rfc724_mid} is already in some chat or deleted."
);
if mime_parser.incoming {
return Ok(None);
}
// For the case if we missed a successful SMTP response. Be optimistic that the message is
// delivered also.
let self_addr = context.get_primary_self_addr().await?;
@@ -543,16 +551,6 @@ pub(crate) async fn receive_imf_inner(
if !msg_has_pending_smtp_job(context, msg_id).await? {
msg_id.set_delivered(context).await?;
}
}
// These checks must be done before processing of SecureJoin and other special messages.
if mime_parser.pre_message == mimeparser::PreMessageMode::Post {
// Post-Message just replaces the attachment and modifies Params, not the whole message.
// This is done in the `handle_post_message` method.
} else if msg_id.is_some() {
info!(
context,
"Message {rfc724_mid} is already in some chat or deleted."
);
return Ok(None);
}
@@ -808,6 +806,8 @@ UPDATE config SET value=? WHERE keyname='configured_addr' AND value!=?1
if transport_changed {
info!(context, "Primary transport changed to {from_addr:?}.");
context.sql.uncache_raw_config("configured_addr").await;
// Regenerate User ID in V4 keys.
context.self_public_key.lock().await.take();
context.emit_event(EventType::TransportsModified);
@@ -2506,10 +2506,7 @@ WHERE id=?
part.typ,
part.bytes as isize,
part.error.as_deref().unwrap_or_default(),
match mime_parser.incoming {
true => state,
false => MessageState::Undefined,
},
state,
DownloadState::Done as u32,
original_msg.id,
),

View File

@@ -5592,27 +5592,27 @@ async fn test_mark_message_as_delivered_only_after_sent_out_fully() -> Result<()
.await
.unwrap();
let (post_msg_id, post_msg_payload) = first_row_in_smtp_queue(alice).await;
assert_eq!(msg_id, post_msg_id);
assert!(post_msg_payload.len() > file_bytes.len());
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
// Alice receives her own post-message because of bcc_self
// This should not yet mark the message as delivered,
// because not everything was sent,
// but it does remove the post-message from the SMTP queue.
receive_imf(alice, post_msg_payload.as_bytes(), false).await?;
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
let (pre_msg_id, pre_msg_payload) = first_row_in_smtp_queue(alice).await;
assert_eq!(msg_id, pre_msg_id);
assert!(pre_msg_payload.len() < file_bytes.len());
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
// Alice receives her own pre-message because of bcc_self
// This should not yet mark the message as delivered,
// because not everything was sent,
// but it does remove the pre-message from the SMTP queue
receive_imf(alice, pre_msg_payload.as_bytes(), false).await?;
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
let (post_msg_id, post_msg_payload) = first_row_in_smtp_queue(alice).await;
assert_eq!(msg_id, post_msg_id);
assert!(post_msg_payload.len() > file_bytes.len());
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutPending);
// Alice receives her own post-message because of bcc_self
// This should now mark the message as delivered,
// because everything was sent by now.
receive_imf(alice, pre_msg_payload.as_bytes(), false).await?;
receive_imf(alice, post_msg_payload.as_bytes(), false).await?;
assert_eq!(msg_id.get_state(alice).await?, MessageState::OutDelivered);
Ok(())

View File

@@ -379,7 +379,7 @@ pub(crate) async fn send_msg_to_smtp(
if retries > 6 {
context
.sql
.execute("DELETE FROM smtp WHERE msg_id=?", (msg_id,))
.execute("DELETE FROM smtp WHERE id=?", (rowid,))
.await
.context("Failed to remove message with exceeded retry limit from smtp table")?;
if let Some(mut msg) = Message::load_from_db_optional(context, msg_id).await? {

View File

@@ -2373,27 +2373,6 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT;
.await?;
}
inc_and_check(&mut migration_version, 152)?;
if dbversion < migration_version {
sql.execute_migration(
"
UPDATE msgs SET state=26 WHERE state=28; -- Change OutMdnRcvd to OutDelivered.
UPDATE msgs SET state=19 WHERE state=24; -- Change OutPreparing to OutFailed.
",
migration_version,
)
.await?;
}
inc_and_check(&mut migration_version, 153)?;
if dbversion < migration_version {
sql.execute_migration(
"CREATE INDEX smtp_index_msg_id ON smtp (msg_id, id)",
migration_version,
)
.await?;
}
let new_version = sql
.get_raw_config_int(VERSION_CFG)
.await?

View File

@@ -29,6 +29,7 @@ const STATISTICS_BOT_VCARD: &str = include_str!("../assets/statistics-bot.vcf");
const SENDING_INTERVAL_SECONDS: i64 = 3600 * 24 * 7; // 1 week
// const SENDING_INTERVAL_SECONDS: i64 = 60; // 1 minute (for testing)
const MESSAGE_STATS_UPDATE_INTERVAL_SECONDS: i64 = 4 * 60; // 4 minutes (less than the lowest ephemeral messages timeout)
const KEY_CREATE_TIMESTAMP_RESOLUTION: u32 = 3600 * 24 * 7 * 4; // 4 weeks
#[derive(Serialize)]
struct Statistics {
@@ -348,7 +349,13 @@ async fn get_stats(context: &Context) -> Result<String> {
let key_create_timestamps: Vec<u32> = load_self_public_keyring(context)
.await?
.iter()
.map(|k| k.created_at().as_secs())
.map(|k| {
k.created_at()
.as_secs()
.div_ceil(KEY_CREATE_TIMESTAMP_RESOLUTION)
.checked_mul(KEY_CREATE_TIMESTAMP_RESOLUTION)
.unwrap_or(0)
})
.collect();
let sending_enabled_timestamps =

View File

@@ -486,6 +486,17 @@ async fn test_stats_key_creation_timestamp() -> Result<()> {
// Alice uses a pregenerated key. It was created at this timestamp:
const ALICE_KEY_CREATION_TIME: u128 = 1582855645;
// The key creation time's resolution is reduced in order to prevent deanonymization:
const CENSORED_KEY_CREATION_TIME: u128 = 1584576000;
assert!(CENSORED_KEY_CREATION_TIME.is_multiple_of(u128::from(KEY_CREATE_TIMESTAMP_RESOLUTION)));
const {
assert!(CENSORED_KEY_CREATION_TIME > ALICE_KEY_CREATION_TIME);
}
assert!(
CENSORED_KEY_CREATION_TIME - ALICE_KEY_CREATION_TIME
< u128::from(KEY_CREATE_TIMESTAMP_RESOLUTION)
);
let alice = &TestContext::new_alice().await;
alice.set_config_bool(Config::StatsSending, true).await?;
@@ -495,7 +506,7 @@ async fn test_stats_key_creation_timestamp() -> Result<()> {
assert_eq!(
key_create_timestamps,
&vec![Value::Number(
Number::from_u128(ALICE_KEY_CREATION_TIME).unwrap()
Number::from_u128(CENSORED_KEY_CREATION_TIME).unwrap()
)]
);

View File

@@ -722,14 +722,12 @@ ORDER BY id"
})
}
/// Returns `SentMessage` instances representing `smtp` rows for the given message. Returned
/// items go in reverse order for historical reasons.
pub async fn get_smtp_rows_for_msg<'a>(&'a self, msg_id: MsgId) -> Vec<SentMessage<'a>> {
let sent_msgs = self
.ctx
.sql
.query_map_vec(
"SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=? ORDER BY id DESC",
"SELECT id, msg_id, mime, recipients FROM smtp WHERE msg_id=?",
(msg_id,),
|row| {
let _id: MsgId = row.get(0)?;
@@ -1057,17 +1055,9 @@ ORDER BY id"
/// This is not hooked up to any SMTP-IMAP pipeline, so the other account must call
/// [`TestContext::recv_msg`] with the returned [`SentMessage`] if it wants to receive
/// the message.
///
/// Removes SMTP jobs existed before and marks the corresponding messages as delivered, as
/// tracking of these jobs is probably already lost by the test code.
pub async fn send_msg(&self, chat_id: ChatId, msg: &mut Message) -> SentMessage<'_> {
while self.pop_sent_msg_opt(Duration::ZERO).await.is_some() {}
let msg_id = chat::send_msg(self, chat_id, msg).await.unwrap();
let rev_order = false;
let res = self
.pop_sent_msg_ex(rev_order, Duration::ZERO)
.await
.unwrap();
let res = self.pop_sent_msg().await;
assert_eq!(
res.sender_msg_id, msg_id,
"Apparently the message was not actually sent out"

View File

@@ -1,6 +1,4 @@
//! Tests about forwarding and saving Pre-Messages
use std::time::Duration;
use anyhow::Result;
use pretty_assertions::assert_eq;
@@ -10,7 +8,6 @@ use crate::chatlist::get_last_message_for_chat;
use crate::download::{DownloadState, PRE_MSG_ATTACHMENT_SIZE_THRESHOLD};
use crate::message::{Message, Viewtype};
use crate::test_utils::TestContextManager;
use crate::tests::pre_messages::util::send_large_file_message;
/// Test that forwarding Pre-Message should forward additional text to not be empty
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -89,43 +86,6 @@ async fn test_forwarding_pre_message_empty_text() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_receive_both() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = &tcm.alice().await;
let bob = &tcm.bob().await;
let alice_chat_id = alice.create_group_with_members("", &[bob]).await;
let (pre_message, post_message, alice_msg_id) =
send_large_file_message(alice, alice_chat_id, Viewtype::File, &vec![0u8; 200_000]).await?;
let msg = bob.recv_msg(&pre_message).await;
let _ = bob.recv_msg_trash(&post_message).await;
let msg = Message::load_from_db(bob, msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.text, "test".to_owned());
forward_msgs(alice, &[alice_msg_id], alice_chat_id).await?;
let rev_order = true;
let msg = bob
.recv_msg(
&alice
.pop_sent_msg_ex(rev_order, Duration::ZERO)
.await
.unwrap(),
)
.await;
assert_eq!(msg.download_state(), DownloadState::Available);
assert_eq!(msg.is_forwarded(), true);
assert_eq!(msg.text, "test".to_owned());
let _ = bob.recv_msg_trash(&alice.pop_sent_msg().await).await;
let msg = Message::load_from_db(bob, msg.id).await?;
assert_eq!(msg.download_state(), DownloadState::Done);
assert_eq!(msg.is_forwarded(), true);
assert_eq!(msg.text, "test".to_owned());
Ok(())
}
/// Test that forwarding Pre-Message should forward additional text to not be empty
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_saving_pre_message_empty_text() -> Result<()> {

View File

@@ -547,8 +547,8 @@ async fn test_webxdc_updates_in_post_message_after_pre_message() -> Result<()> {
.await?;
send_msg(alice, alice_chat_id, &mut alice_instance).await?;
let pre_message = alice.pop_sent_msg().await;
let post_message = alice.pop_sent_msg().await;
let pre_message = alice.pop_sent_msg().await;
let bob_instance = bob.recv_msg(&pre_message).await;
assert_eq!(bob_instance.download_state, DownloadState::Available);
@@ -588,8 +588,8 @@ async fn test_webxdc_updates_in_post_message_without_pre_message() -> Result<()>
.await?;
send_msg(alice, alice_chat_id, &mut alice_instance).await?;
let pre_message = alice.pop_sent_msg().await;
let post_message = alice.pop_sent_msg().await;
let pre_message = alice.pop_sent_msg().await;
// Bob receives post-message first.
let bob_instance = bob.recv_msg(&post_message).await;

View File

@@ -791,18 +791,7 @@ pub(crate) async fn sync_transports(
context
.sql
.transaction(|transaction| {
let configured_addr = transaction.query_row(
"SELECT value FROM config WHERE keyname='configured_addr'",
(),
|row| {
let addr: String = row.get(0)?;
Ok(addr)
},
)?;
for RemovedTransportData { addr, timestamp } in removed_transports {
if *addr == configured_addr {
continue;
}
modified |= transaction.execute(
"DELETE FROM transports
WHERE addr=? AND add_timestamp<=?",

View File

@@ -550,7 +550,7 @@ impl Context {
let send_now = !matches!(
instance.state,
MessageState::Undefined | MessageState::OutDraft
MessageState::Undefined | MessageState::OutPreparing | MessageState::OutDraft
);
status_update.uid = Some(create_id());