Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-28 02:46:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Mark messages as seen in IMAP loop

Browse Source 
MarkseenMsgOnImap job, that was responsible for marking messages as
seen on IMAP and sending MDNs, has been removed.

Messages waiting to be marked as seen are now stored in a
single-column imap_markseen table consisting of foreign keys pointing
to corresponding imap table records.

Messages are marked as seen in batches in the inbox loop. UIDs are
grouped by folders to reduce the number of requests, including folder
selection requests. UID grouping logic has been factored out of
move_delete_messages into UidGrouper iterator to avoid code duplication.

Messages are marked as seen right before fetching from the inbox
folder. This ensures that even if new messages arrive into inbox while
the connection has another folder selected to mark messages there, all
messages are fetched before going IDLE. Ideally marking messages as
seen should be done after fetching and moving, as it is a low-priority
task, but this requires skipping IDLE if UIDNEXT has advanced since
previous time inbox has been selected. This is outside of the scope of
this change.

MDNs are now queued independently of marking the messages as seen.
SendMdn job is created directly rather than after marking the message
as seen on IMAP. Previously sending MDNs was done in MarkseenMsgOnImap
avoid duplicate MDN sending by setting $MDNSent flag together with
\Seen flag and skipping MDN sending if the flag is already set. This
is not the case anymore as $MDNSent flag support has been removed in
9c077c98cd and duplicate MDN sending in
multi-device case is avoided by synchronizing Seen status since
833e5f46cc as long as the server
supports CONDSTORE extension.
This commit is contained in:
link2xt
2022-04-16 11:19:28 +00:00
parent 213e67dea2
commit 66c4de2607
8 changed files with 217 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
src/imap.rs
View File

@@ -7,6 +7,7 @@ use std::{
cmp,
cmp::max,
collections::{BTreeMap, BTreeSet},
iter::Peekable,
};
use anyhow::{bail, format_err, Context as _, Result};
@@ -31,14 +32,13 @@ use crate::dc_receive_imf::{
use crate::dc_tools::dc_create_id;
use crate::events::EventType;
use crate::headerdef::{HeaderDef, HeaderDefMap};
use crate::job::{self, Action};
use crate::job;
use crate::login_param::{
CertificateChecks, LoginParam, ServerAddress, ServerLoginParam, Socks5Config,
};
use crate::message::{self, Message, MessageState, MessengerMessage, MsgId, Viewtype};
use crate::mimeparser;
use crate::oauth2::dc_get_oauth2_access_token;
use crate::param::Params;
use crate::provider::Socket;
use crate::scheduler::connectivity::ConnectivityStore;
use crate::scheduler::InterruptInfo;
@@ -165,6 +165,67 @@ struct ImapConfig {
pub can_condstore: bool,
}
struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
inner: Peekable<T>,
}
impl<T, I> From<I> for UidGrouper<T>
where
T: Iterator<Item = (i64, u32, String)>,
I: IntoIterator<IntoIter = T>,
{
fn from(inner: I) -> Self {
Self {
inner: inner.into_iter().peekable(),
}
}
}
impl<T: Iterator<Item = (i64, u32, String)>> Iterator for UidGrouper<T> {
// Tuple of folder, row IDs, and UID range as a string.
type Item = (String, Vec<i64>, String);
fn next(&mut self) -> Option<Self::Item> {
let (_, _, folder) = self.inner.peek().cloned()?;
let mut uid_set = String::new();
let mut rowid_set = Vec::new();
while uid_set.len() < 1000 {
// Construct a new range.
if let Some((start_rowid, start_uid, _)) = self
.inner
.next_if(|(_, _, start_folder)| start_folder == &folder)
{
rowid_set.push(start_rowid);
let mut end_uid = start_uid;
while let Some((next_rowid, next_uid, _)) =
self.inner.next_if(|(_, next_uid, next_folder)| {
next_folder == &folder && *next_uid == end_uid + 1
})
{
end_uid = next_uid;
rowid_set.push(next_rowid);
}
let uid_range = UidRange {
start: start_uid,
end: end_uid,
};
if !uid_set.is_empty() {
uid_set.push(',');
}
uid_set.push_str(&uid_range.to_string());
} else {
break;
}
}
Some((folder, rowid_set, uid_set))
}
}
impl Imap {
/// Creates new disconnected IMAP client using the specific login parameters.
///
@@ -944,7 +1005,7 @@ impl Imap {
///
/// This is the only place where messages are moved or deleted on the IMAP server.
async fn move_delete_messages(&mut self, context: &Context, folder: &str) -> Result<()> {
let mut rows = context
let rows = context
.sql
.query_map(
"SELECT id, uid, target FROM imap
@@ -960,48 +1021,12 @@ impl Imap {
},
|rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
)
.await?
.into_iter()
.peekable();
.await?;
self.prepare(context).await?;
self.select_folder(context, Some(folder)).await?;
while let Some((_, _, target)) = rows.peek().cloned() {
// Construct next request for the target folder.
let mut uid_set = String::new();
let mut rowid_set = Vec::new();
while uid_set.len() < 1000 {
// Construct a new range.
if let Some((start_rowid, start_uid, _)) =
rows.next_if(|(_, _, start_target)| start_target == &target)
{
rowid_set.push(start_rowid);
let mut end_uid = start_uid;
while let Some((next_rowid, next_uid, _)) =
rows.next_if(|(_, next_uid, next_target)| {
next_target == &target && *next_uid == end_uid + 1
})
{
end_uid = next_uid;
rowid_set.push(next_rowid);
}
let uid_range = UidRange {
start: start_uid,
end: end_uid,
};
if !uid_set.is_empty() {
uid_set.push(',');
}
uid_set.push_str(&uid_range.to_string());
} else {
break;
}
}
for (target, rowid_set, uid_set) in UidGrouper::from(rows) {
// Empty target folder name means messages should be deleted.
if target.is_empty() {
self.delete_message_batch(context, &uid_set, rowid_set)
@@ -1028,6 +1053,62 @@ impl Imap {
Ok(())
}
/// Stores pending `\Seen` flags for messages in `imap_markseen` table.
pub(crate) async fn store_seen_flags(&mut self, context: &Context) -> Result<()> {
self.prepare(context).await?;
let rows = context
.sql
.query_map(
"SELECT imap.id, uid, folder FROM imap, imap_markseen
WHERE imap.id = imap_markseen.id AND target = folder
ORDER BY folder, uid",
[],
|row| {
let rowid: i64 = row.get(0)?;
let uid: u32 = row.get(1)?;
let folder: String = row.get(2)?;
Ok((rowid, uid, folder))
},
|rows| rows.collect::<Result<Vec<_>, _>>().map_err(Into::into),
)
.await?;
for (folder, rowid_set, uid_set) in UidGrouper::from(rows) {
self.select_folder(context, Some(&folder))
.await
.context("failed to select folder")?;
if let Err(err) = self.add_flag_finalized_with_set(&uid_set, "\\Seen").await {
warn!(
context,
"Cannot mark messages {} in folder {} as seen, will retry later: {}.",
uid_set,
folder,
err
);
} else {
info!(
context,
"Marked messages {} in folder {} as seen.", uid_set, folder
);
context
.sql
.execute(
format!(
"DELETE FROM imap_markseen WHERE id IN ({})",
sql::repeat_vars(rowid_set.len())?
),
rusqlite::params_from_iter(rowid_set),
)
.await
.context("cannot remove messages marked as seen from imap_markseen table")?;
}
}
Ok(())
}
/// Synchronizes `\Seen` flags using `CONDSTORE` extension.
pub(crate) async fn sync_seen_flags(&mut self, context: &Context, folder: &str) -> Result<()> {
if !self.config.can_condstore {
@@ -1364,11 +1445,6 @@ impl Imap {
/// the flag, or other imap-errors, returns true as well.
///
/// Returning error means that the operation can be retried.
async fn add_flag_finalized(&mut self, server_uid: u32, flag: &str) -> Result<()> {
let s = server_uid.to_string();
self.add_flag_finalized_with_set(&s, flag).await
}
async fn add_flag_finalized_with_set(&mut self, uid_set: &str, flag: &str) -> Result<()> {
if self.should_reconnect() {
bail!("Can't set flag, should reconnect");
@@ -1386,7 +1462,7 @@ impl Imap {
Ok(())
}
pub async fn prepare_imap_operation_on_msg(
pub(crate) async fn prepare_imap_operation_on_msg(
&mut self,
context: &Context,
folder: &str,
@@ -1426,32 +1502,6 @@ impl Imap {
}
}
pub(crate) async fn set_seen(
&mut self,
context: &Context,
folder: &str,
uid: u32,
) -> ImapActionResult {
if let Some(imapresult) = self
.prepare_imap_operation_on_msg(context, folder, uid)
.await
{
return imapresult;
}
// we are connected, and the folder is selected
info!(context, "Marking message {}/{} as seen...", folder, uid,);
if let Err(err) = self.add_flag_finalized(uid, "\\Seen").await {
warn!(
context,
"Cannot mark message {} in folder {} as seen, ignoring: {}.", uid, folder, err
);
ImapActionResult::Failed
} else {
ImapActionResult::Success
}
}
pub async fn ensure_configured_folders(
&mut self,
context: &Context,
@@ -1882,13 +1932,11 @@ pub(crate) async fn prefetch_should_download(
mut flags: impl Iterator<Item = Flag<'_>>,
show_emails: ShowEmails,
) -> Result<bool> {
if let Some(msg_id) = message::rfc724_mid_exists(context, message_id).await? {
// We know the Message-ID already, it must be a Bcc: to self.
job::add(
context,
job::Job::new(Action::MarkseenMsgOnImap, msg_id.to_u32(), Params::new(), 0),
)
.await?;
if message::rfc724_mid_exists(context, message_id)
.await?
.is_some()
{
markseen_on_imap(context, message_id).await?;
return Ok(false);
}
@@ -2022,6 +2070,20 @@ async fn mark_seen_by_uid(
}
}
pub(crate) async fn markseen_on_imap(context: &Context, message_id: &str) -> Result<()> {
context
.sql
.execute(
"INSERT OR IGNORE INTO imap_markseen (id)
SELECT id FROM imap WHERE rfc724_mid=?",
paramsv![message_id],
)
.await?;
context.interrupt_inbox(InterruptInfo::new(false)).await;
Ok(())
}
/// uid_next is the next unique identifier value from the last time we fetched a folder
/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
/// This function is used to update our uid_next after fetching messages.