Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-29 11:26:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

test: remove fetch_existing tests

Browse Source 
fetch_existing option is not enabled in existing clients
and does not work with encrypted messages
without importing the key into a newely created account.
This commit is contained in:
link2xt
2025-04-02 18:01:02 +00:00
committed by l
parent c239da542c
commit f942a63c5d
11 changed files with 9 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
python/tests/test_0_complex_or_slow.py
View File

@@ -1,7 +1,6 @@
import sys
import time
import pytest
import deltachat as dc
@@ -196,118 +195,6 @@ def test_qr_verified_group_and_chatting(acfactory, lp):
assert msg.is_encrypted()
@pytest.mark.parametrize("mvbox_move", [False, True])
def test_fetch_existing(acfactory, lp, mvbox_move):
"""Delta Chat reads the recipients from old emails sent by the user and adds them as contacts.
This way, we can already offer them some email addresses they can write to.
Also, the newest existing emails from each folder are fetched during onboarding.
Additionally tests that bcc_self messages moved to the mvbox/sentbox are marked as read."""
def assert_folders_configured(ac):
"""There was a bug that scan_folders() set the configured folders to None under some circumstances.
So, check that they are still configured:"""
assert ac.get_config("configured_sentbox_folder") == "Sent"
if mvbox_move:
assert ac.get_config("configured_mvbox_folder")
ac1 = acfactory.new_online_configuring_account(mvbox_move=mvbox_move)
ac2 = acfactory.new_online_configuring_account()
acfactory.wait_configured(ac1)
ac1.direct_imap.create_folder("Sent")
ac1.set_config("sentbox_watch", "1")
# We need to reconfigure to find the new "Sent" folder.
# `scan_folders()`, which runs automatically shortly after `start_io()` is invoked,
# would also find the "Sent" folder, but it would be too late:
# The sentbox thread, started by `start_io()`, would have seen that there is no
# ConfiguredSentboxFolder and do nothing.
acfactory._acsetup.start_configure(ac1)
acfactory.bring_accounts_online()
assert_folders_configured(ac1)
lp.sec("send out message with bcc to ourselves")
ac1.set_config("bcc_self", "1")
chat = acfactory.get_accepted_chat(ac1, ac2)
chat.send_text("message text")
lp.sec("wait until the bcc_self message arrives in correct folder and is marked seen")
if mvbox_move:
ac1._evtracker.get_info_contains("Marked messages [0-9]+ in folder DeltaChat as seen.")
else:
ac1._evtracker.get_info_contains("Marked messages [0-9]+ in folder INBOX as seen.")
assert_folders_configured(ac1)
lp.sec("create a cloned ac1 and fetch contact history during configure")
ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
ac1_clone.set_config("fetch_existing_msgs", "1")
acfactory.wait_configured(ac1_clone)
ac1_clone.start_io()
assert_folders_configured(ac1_clone)
lp.sec("check that ac2 contact was fetched during configure")
ac1_clone._evtracker.get_matching("DC_EVENT_CONTACTS_CHANGED")
ac2_addr = ac2.get_config("addr")
assert any(c.addr == ac2_addr for c in ac1_clone.get_contacts())
assert_folders_configured(ac1_clone)
lp.sec("check that messages changed events arrive for the correct message")
msg = ac1_clone._evtracker.wait_next_messages_changed()
assert msg.text == "message text"
assert_folders_configured(ac1)
assert_folders_configured(ac1_clone)
def test_fetch_existing_msgs_group_and_single(acfactory, lp):
"""There was a bug concerning fetch-existing-msgs:
A sent a message to you, adding you to a group. This created a contact request.
You wrote a message to A, creating a chat.
...but the group stayed blocked.
So, after fetch-existing-msgs you have one contact request and one chat with the same person.
See https://github.com/deltachat/deltachat-core-rust/issues/2097"""
ac1 = acfactory.new_online_configuring_account()
ac2 = acfactory.new_online_configuring_account()
acfactory.bring_accounts_online()
lp.sec("receive a message")
ac2.create_group_chat("group name", contacts=[ac1]).send_text("incoming, unencrypted group message")
ac1._evtracker.wait_next_incoming_message()
lp.sec("send out message with bcc to ourselves")
ac1.set_config("bcc_self", "1")
ac1_ac2_chat = ac1.create_chat(ac2)
ac1_ac2_chat.send_text("outgoing, encrypted direct message, creating a chat")
# wait until the bcc_self message arrives
ac1._evtracker.get_info_contains("Marked messages [0-9]+ in folder INBOX as seen.")
lp.sec("Clone online account and let it fetch the existing messages")
ac1_clone = acfactory.new_online_configuring_account(cloned_from=ac1)
ac1_clone.set_config("fetch_existing_msgs", "1")
acfactory.wait_configured(ac1_clone)
ac1_clone.start_io()
ac1_clone._evtracker.wait_idle_inbox_ready()
chats = ac1_clone.get_chats()
assert len(chats) == 4 # two newly created chats + self-chat + device-chat
group_chat = [c for c in chats if c.get_name() == "group name"][0]
assert group_chat.is_group()
(private_chat,) = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
assert not private_chat.is_group()
group_messages = group_chat.get_messages()
assert len(group_messages) == 1
assert group_messages[0].text == "incoming, unencrypted group message"
private_messages = private_chat.get_messages()
# We can't decrypt the message in this chat, so the chat is empty:
assert len(private_messages) == 0
def test_undecipherable_group(acfactory, lp):
"""Test how group messages that cannot be decrypted are
handled.

7
src/config.rs
View File

@@ -182,12 +182,6 @@ pub enum Config {
#[strum(props(default = "0"))] // also change MediaQuality.default() on changes
MediaQuality,
/// If set to "1", on the first time `start_io()` is called after configuring,
/// the newest existing messages are fetched.
/// Existing recipients are added to the contact database regardless of this setting.
#[strum(props(default = "0"))]
FetchExistingMsgs,
/// If set to "1", then existing messages are considered to be already fetched.
/// This flag is reset after successful configuration.
#[strum(props(default = "1"))]
@@ -707,7 +701,6 @@ impl Context {
| Config::SentboxWatch
| Config::MvboxMove
| Config::OnlyFetchMvbox
| Config::FetchExistingMsgs
| Config::DeleteToTrash
| Config::Configured
| Config::Bot

3
src/constants.rs
View File

@@ -180,9 +180,6 @@ pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
/// if none of these flags are set, the default is chosen
pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
/// How many existing messages shall be fetched after configuration.
pub(crate) const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
// max. weight of images to send w/o recoding
pub const BALANCED_IMAGE_BYTES: usize = 500_000;
pub const WORSE_IMAGE_BYTES: usize = 130_000;

6
src/context.rs
View File

@@ -903,12 +903,6 @@ impl Context {
}
res.insert("secondary_addrs", secondary_addrs);
res.insert(
"fetch_existing_msgs",
self.get_config_int(Config::FetchExistingMsgs)
.await?
.to_string(),
);
res.insert(
"fetched_existing_msgs",
self.get_config_bool(Config::FetchedExistingMsgs)

18
src/download.rs
View File

@@ -220,7 +220,6 @@ impl Session {
vec![uid],
&uid_message_ids,
false,
false,
)
.await?;
if last_uid.is_none() {
@@ -369,7 +368,6 @@ mod tests {
header.as_bytes(),
false,
Some(100000),
false,
)
.await?;
let msg = t.get_last_msg().await;
@@ -385,7 +383,6 @@ mod tests {
format!("{header}\n\n100k text...").as_bytes(),
false,
None,
false,
)
.await?;
let msg = t.get_last_msg().await;
@@ -420,7 +417,6 @@ mod tests {
Content-Type: text/plain",
false,
Some(100000),
false,
)
.await?;
assert_eq!(
@@ -457,7 +453,6 @@ mod tests {
sent2.payload().as_bytes(),
false,
Some(sent2.payload().len() as u32),
false,
)
.await?;
let msg = bob.get_last_msg().await;
@@ -473,7 +468,6 @@ mod tests {
sent2.payload().as_bytes(),
false,
None,
false,
)
.await?;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0);
@@ -517,15 +511,7 @@ mod tests {
";
// not downloading the mdn results in an placeholder
receive_imf_from_inbox(
&bob,
"bar@example.org",
raw,
false,
Some(raw.len() as u32),
false,
)
.await?;
receive_imf_from_inbox(&bob, "bar@example.org", raw, false, Some(raw.len() as u32)).await?;
let msg = bob.get_last_msg().await;
let chat_id = msg.chat_id;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 1);
@@ -533,7 +519,7 @@ mod tests {
// downloading the mdn afterwards expands to nothing and deletes the placeholder directly
// (usually mdn are too small for not being downloaded directly)
receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None, false).await?;
receive_imf_from_inbox(&bob, "bar@example.org", raw, false, None).await?;
assert_eq!(get_chat_msgs(&bob, chat_id).await?.len(), 0);
assert!(Message::load_from_db_optional(&bob, msg.id)
.await?

40
src/imap.rs
View File

@@ -507,7 +507,7 @@ impl Imap {
}
let msgs_fetched = self
.fetch_new_messages(context, session, watch_folder, folder_meaning, false)
.fetch_new_messages(context, session, watch_folder, folder_meaning)
.await
.context("fetch_new_messages")?;
if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
@@ -535,7 +535,6 @@ impl Imap {
session: &mut Session,
folder: &str,
folder_meaning: FolderMeaning,
fetch_existing_msgs: bool,
) -> Result<bool> {
if should_ignore_folder(context, folder, folder_meaning).await? {
info!(context, "Not fetching from {folder:?}.");
@@ -552,7 +551,7 @@ impl Imap {
return Ok(false);
}
if !session.new_mail && !fetch_existing_msgs {
if !session.new_mail {
info!(context, "No new emails in folder {folder:?}.");
return Ok(false);
}
@@ -561,14 +560,7 @@ impl Imap {
let uid_validity = get_uidvalidity(context, folder).await?;
let old_uid_next = get_uid_next(context, folder).await?;
let msgs = if fetch_existing_msgs {
session
.prefetch_existing_msgs()
.await
.context("prefetch_existing_msgs")?
} else {
session.prefetch(old_uid_next).await.context("prefetch")?
};
let msgs = session.prefetch(old_uid_next).await.context("prefetch")?;
let read_cnt = msgs.len();
let download_limit = context.download_limit().await?;
@@ -721,7 +713,6 @@ impl Imap {
uids_fetch_in_batch.split_off(0),
&uid_message_ids,
fetch_partially,
fetch_existing_msgs,
)
.await
.context("fetch_many_msgs")?;
@@ -786,28 +777,6 @@ impl Imap {
.await
.context("failed to get recipients from the inbox")?;
if context.get_config_bool(Config::FetchExistingMsgs).await? {
for meaning in [
FolderMeaning::Mvbox,
FolderMeaning::Inbox,
FolderMeaning::Sent,
] {
let config = match meaning.to_config() {
Some(c) => c,
None => continue,
};
if let Some(folder) = context.get_config(config).await? {
info!(
context,
"Fetching existing messages from folder {folder:?}."
);
self.fetch_new_messages(context, session, &folder, meaning, true)
.await
.context("could not fetch existing messages")?;
}
}
}
info!(context, "Done fetching existing messages.");
Ok(())
}
@@ -1334,7 +1303,6 @@ impl Session {
/// Returns the last UID fetched successfully and the info about each downloaded message.
/// If the message is incorrect or there is a failure to write a message to the database,
/// it is skipped and the error is logged.
#[expect(clippy::too_many_arguments)]
pub(crate) async fn fetch_many_msgs(
&mut self,
context: &Context,
@@ -1343,7 +1311,6 @@ impl Session {
request_uids: Vec<u32>,
uid_message_ids: &BTreeMap<u32, String>,
fetch_partially: bool,
fetching_existing_messages: bool,
) -> Result<(Option<u32>, Vec<ReceivedMsg>)> {
let mut last_uid = None;
let mut received_msgs = Vec::new();
@@ -1477,7 +1444,6 @@ impl Session {
body,
is_seen,
partial,
fetching_existing_messages,
)
.await
{

31
src/imap/session.rs
View File

@@ -1,4 +1,3 @@
use std::cmp;
use std::collections::BTreeMap;
use std::ops::{Deref, DerefMut};
@@ -7,7 +6,6 @@ use async_imap::types::Mailbox;
use async_imap::Session as ImapSession;
use futures::TryStreamExt;
use crate::constants::DC_FETCH_EXISTING_MSGS_COUNT;
use crate::imap::capabilities::Capabilities;
use crate::net::session::SessionStream;
@@ -143,33 +141,4 @@ impl Session {
Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
}
/// Like prefetch(), but not for new messages but existing ones (the DC_FETCH_EXISTING_MSGS_COUNT newest messages)
pub(crate) async fn prefetch_existing_msgs(
&mut self,
) -> Result<Vec<(u32, async_imap::types::Fetch)>> {
let exists: i64 = {
let mailbox = self.selected_mailbox.as_ref().context("no mailbox")?;
mailbox.exists.into()
};
// Fetch last DC_FETCH_EXISTING_MSGS_COUNT (100) messages.
// Sequence numbers are sequential. If there are 1000 messages in the inbox,
// we can fetch the sequence numbers 900-1000 and get the last 100 messages.
let first = cmp::max(1, exists - DC_FETCH_EXISTING_MSGS_COUNT + 1);
let set = format!("{first}:{exists}");
let mut list = self
.fetch(&set, PREFETCH_FLAGS)
.await
.context("IMAP Could not fetch")?;
let mut msgs = BTreeMap::new();
while let Some(msg) = list.try_next().await? {
if let Some(msg_uid) = msg.uid {
msgs.insert((msg.internal_date(), msg_uid), msg);
}
}
Ok(msgs.into_iter().map(|((_, uid), msg)| (uid, msg)).collect())
}
}

2
src/reaction.rs
View File

@@ -900,7 +900,6 @@ Here's my footer -- bob@example.net"
msg_header.as_bytes(),
false,
Some(100000),
false,
)
.await?
.unwrap();
@@ -931,7 +930,6 @@ Here's my footer -- bob@example.net"
msg_full.as_bytes(),
false,
None,
false,
)
.await?;

26
src/receive_imf.rs
View File

@@ -98,12 +98,11 @@ pub async fn receive_imf(
head.as_bytes(),
seen,
Some(imf_raw.len().try_into()?),
false,
)
.await;
}
}
receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None, false).await
receive_imf_from_inbox(context, &rfc724_mid, imf_raw, seen, None).await
}
/// Emulates reception of a message from "INBOX".
@@ -116,7 +115,6 @@ pub(crate) async fn receive_imf_from_inbox(
imf_raw: &[u8],
seen: bool,
is_partial_download: Option<u32>,
fetching_existing_messages: bool,
) -> Result<Option<ReceivedMsg>> {
receive_imf_inner(
context,
@@ -127,7 +125,6 @@ pub(crate) async fn receive_imf_from_inbox(
imf_raw,
seen,
is_partial_download,
fetching_existing_messages,
)
.await
}
@@ -171,7 +168,6 @@ pub(crate) async fn receive_imf_inner(
imf_raw: &[u8],
seen: bool,
is_partial_download: Option<u32>,
fetching_existing_messages: bool,
) -> Result<Option<ReceivedMsg>> {
if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
info!(
@@ -444,7 +440,6 @@ pub(crate) async fn receive_imf_inner(
seen,
is_partial_download,
replace_msg_id,
fetching_existing_messages,
prevent_rename,
verified_encryption,
)
@@ -718,13 +713,10 @@ async fn add_parts(
seen: bool,
is_partial_download: Option<u32>,
mut replace_msg_id: Option<MsgId>,
fetching_existing_messages: bool,
prevent_rename: bool,
verified_encryption: VerifiedEncryption,
) -> Result<ReceivedMsg> {
let is_bot = context.get_config_bool(Config::Bot).await?;
// Bots handle existing messages the same way as new ones.
let fetching_existing_messages = fetching_existing_messages && !is_bot;
let rfc724_mid_orig = &mime_parser
.get_rfc724_mid()
.unwrap_or(rfc724_mid.to_string());
@@ -1045,11 +1037,7 @@ async fn add_parts(
}
}
state = if seen
|| fetching_existing_messages
|| is_mdn
|| chat_id_blocked == Blocked::Yes
|| group_changes.silent
state = if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent
// No check for `hidden` because only reactions are such and they should be `InFresh`.
{
MessageState::InSeen
@@ -1116,7 +1104,7 @@ async fn add_parts(
}
}
if mime_parser.decrypting_failed && !fetching_existing_messages {
if mime_parser.decrypting_failed {
if chat_id.is_none() {
chat_id = Some(DC_CHAT_ID_TRASH);
} else {
@@ -1242,12 +1230,6 @@ async fn add_parts(
}
}
if fetching_existing_messages && mime_parser.decrypting_failed {
chat_id = Some(DC_CHAT_ID_TRASH);
// We are only gathering old messages on first start. We do not want to add loads of non-decryptable messages to the chats.
info!(context, "Existing non-decipherable message (TRASH).");
}
if mime_parser.webxdc_status_update.is_some() && mime_parser.parts.len() == 1 {
if let Some(part) = mime_parser.parts.first() {
if part.typ == Viewtype::Text && part.msg.is_empty() {
@@ -1510,7 +1492,7 @@ async fn add_parts(
while let Some(part) = parts.next() {
if part.is_reaction {
let reaction_str = simplify::remove_footers(part.msg.as_str());
let is_incoming_fresh = mime_parser.incoming && !seen && !fetching_existing_messages;
let is_incoming_fresh = mime_parser.incoming && !seen;
set_msg_reaction(
context,
mime_in_reply_to,

42
src/receive_imf/receive_imf_tests.rs
View File

@@ -3466,46 +3466,6 @@ async fn test_send_as_bot() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_bot_recv_existing_msg() -> Result<()> {
let mut tcm = TestContextManager::new();
let bob = &tcm.bob().await;
bob.set_config(Config::Bot, Some("1")).await.unwrap();
bob.set_config(Config::FetchExistingMsgs, Some("1"))
.await
.unwrap();
let fetching_existing_messages = true;
let msg = receive_imf_from_inbox(
bob,
"first@example.org",
b"From: Alice <alice@example.org>\n\
To: Bob <bob@example.net>\n\
Chat-Version: 1.0\n\
Message-ID: <first@example.org>\n\
Date: Sun, 14 Nov 2021 00:10:00 +0000\n\
Content-Type: text/plain\n\
\n\
hello\n",
false,
None,
fetching_existing_messages,
)
.await?
.unwrap();
let msg = Message::load_from_db(bob, msg.msg_ids[0]).await?;
assert_eq!(msg.state, MessageState::InFresh);
let event = bob
.evtracker
.get_matching(|ev| matches!(ev, EventType::IncomingMsg { .. }))
.await;
let EventType::IncomingMsg { chat_id, msg_id } = event else {
unreachable!();
};
assert_eq!(chat_id, msg.chat_id);
assert_eq!(msg_id, msg.id);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_wrong_date_in_imf_section() {
let mut tcm = TestContextManager::new();
@@ -4812,7 +4772,6 @@ Content-Type: text/plain
Chat-Group-Member-Added: charlie@example.com",
false,
Some(100000),
false,
)
.await?
.context("no received message")?;
@@ -4850,7 +4809,6 @@ Content-Type: text/plain
Chat-Group-Member-Added: charlie@example.com",
false,
None,
false,
)
.await?
.context("no received message")?;

2
src/webxdc/webxdc_tests.rs
View File

@@ -339,7 +339,6 @@ async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
sent1.payload().as_bytes(),
false,
Some(70790),
false,
)
.await?;
let bob_instance = bob.get_last_msg().await;
@@ -354,7 +353,6 @@ async fn test_webxdc_update_for_not_downloaded_instance() -> Result<()> {
sent1.payload().as_bytes(),
false,
None,
false,
)
.await?
.unwrap();