Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-09-22 13:01:21 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: perform background fetch from all transports

Browse Source 
With I/O stopped, `background_fetch()` connected only to the transport of `configured_addr`
and we now instead fan out to all transports in a controlled loop.
If a first transport finished fetching new messages
cancel all other attempts and return.

This is meant to address the problem that amzd described
where a profile with one functioning and one hanging transport,
shows the first notification, then hangs 15 seconds waiting for the hanging transport.
meanwhile a second NSE arrives and dies, and the second message is not notified
or only generically.

Also drop the quota check from this background fetch path:
its result is in-memory only, discarded when the iOS notification service exits,
and the regular scheduler fetching refreshes it every 60s anyway.
Moreover, quota errors/running full is pretty rare
since relays generally automatically stay under quota these days.
It's another round trip for each transport of each profile and simply not necessary.

Also adds previously missing online tests.
This commit is contained in:
holger krekel
2026-09-03 00:23:00 +00:00
parent 44387f5e58
commit dced877c90
9 changed files with 155 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/constants.rs
View File

@@ -172,9 +172,6 @@ pub const MAX_RCVD_IMAGE_PIXELS: u32 = 50_000_000;
// Relays typically advertise their limit via IMAP METADATA.
pub(crate) const DEFAULT_MAX_SMTP_RCPT_TO: u32 = 50;
/// How far the last quota check needs to be in the past to be checked by the background function (in seconds).
pub(crate) const DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT: u64 = 12 * 60 * 60; // 12 hours
/// How far in the future the sender timestamp of a message is allowed to be, in seconds. Also used
/// in the group membership consistency algo to reject outdated membership changes.
pub(crate) const TIMESTAMP_SENT_TOLERANCE: i64 = 60;

43
src/context.rs
View File

@@ -16,11 +16,11 @@ use tokio::sync::{Mutex, Notify, RwLock};
use crate::chat::{ChatId, get_chat_cnt};
use crate::config::Config;
use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR};
use crate::constants::{self, DC_VERSION_STR};
use crate::contact::{Contact, ContactId};
use crate::debug_logging::DebugLogging;
use crate::events::{Event, EventEmitter, EventType, Events};
use crate::imap::{Imap, ServerMetadata};
use crate::imap::ServerMetadata;
use crate::log::warn;
use crate::logged_debug_assert;
use crate::message::{self, MessageState, MsgId};
@@ -599,56 +599,29 @@ impl Context {
Ok(constants::DEFAULT_MAX_SMTP_RCPT_TO)
}
/// Does a single round of fetching from IMAP and returns.
/// Does a single round of fetching messages from all transports and returns.
///
/// Can be used even if I/O is currently stopped.
/// If I/O is currently stopped, starts a new IMAP connection
/// and fetches from Inbox and DeltaChat folders.
/// If I/O is stopped, fetches over a dedicated connection per transport
/// and returns as soon as one of them fetched messages.
pub async fn background_fetch(&self) -> Result<()> {
if !(self.is_configured().await?) {
return Ok(());
}
let address = self.get_primary_self_addr().await?;
let time_start = tools::Time::now();
info!(self, "background_fetch started fetching {address}.");
info!(self, "background_fetch started.");
if self.scheduler.is_running().await {
self.scheduler.maybe_network().await;
self.wait_for_all_work_done().await;
} else {
// Pause the scheduler to ensure another connection does not start
// while we are fetching on a dedicated connection.
let _pause_guard = self.scheduler.pause(self).await?;
// Start a new dedicated connection.
let mut connection = Imap::new_configured(self, channel::bounded(1).1).await?;
let mut session = connection.prepare(self).await?;
// Fetch IMAP folders.
let folder = connection.folder.clone();
connection
.fetch_move_delete(self, &mut session, &folder)
.await?;
// Update quota (to send warning if full) - but only check it once in a while.
// note: For now this only checks quota of primary transport,
// because background check only checks primary transport at the moment
if self
.quota_needs_update(
session.transport_id(),
DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT,
)
.await
&& let Err(err) = self.update_recent_quota(&mut session, &folder).await
{
warn!(self, "Failed to update quota: {err:#}.");
}
self.scheduler.background_fetch_any(self).await?;
}
info!(
self,
"background_fetch done for {address} took {:?}.",
"background_fetch done, took {:?}.",
time_elapsed(&time_start),
);

37
src/imap.rs
View File

@@ -19,6 +19,7 @@ use async_imap::types::{Fetch, Flag, UnsolicitedResponse};
use futures::{FutureExt as _, TryStreamExt};
use futures_lite::FutureExt;
use ratelimit::Ratelimit;
use tokio_util::sync::CancellationToken;
use url::Url;
use crate::chat::{self, add_device_msg};
@@ -104,6 +105,10 @@ pub(crate) struct Imap {
/// IMAP UID resync request receiver.
pub(crate) resync_request_receiver: async_channel::Receiver<()>,
/// The background fetch is cancelled once messages are fetched from one of the transports,
/// so that the other transports fetch nothing.
pub(crate) background_fetch_stop_token: Option<CancellationToken>,
}
#[derive(Debug, Default)]
@@ -237,21 +242,10 @@ impl Imap {
ratelimit: Ratelimit::new(Duration::new(120, 0), 2.0),
resync_request_sender,
resync_request_receiver,
background_fetch_stop_token: None,
})
}
/// Creates new disconnected IMAP client using configured parameters.
pub async fn new_configured(
context: &Context,
idle_interrupt_receiver: Receiver<()>,
) -> Result<Self> {
let (transport_id, param) = ConfiguredLoginParam::load(context)
.await?
.context("Not configured")?;
let imap = Self::new(context, transport_id, param, idle_interrupt_receiver).await?;
Ok(imap)
}
/// Returns transport ID of the IMAP client.
pub fn transport_id(&self) -> u32 {
self.transport_id
@@ -428,12 +422,14 @@ impl Imap {
///
/// Prefetches headers and downloads new message from the folder, moves messages away from the
/// folder and deletes messages in the folder.
///
/// Returns true if at least one message was fetched.
pub async fn fetch_move_delete(
&mut self,
context: &Context,
session: &mut Session,
watch_folder: &str,
) -> Result<()> {
) -> Result<bool> {
ensure_and_debug_assert!(!watch_folder.is_empty(), "Watched folder cannot be empty");
if !context.sql.is_open().await {
// probably shutdown
@@ -463,7 +459,7 @@ impl Imap {
.await
.context("move_delete_messages")?;
Ok(())
Ok(msgs_fetched)
}
/// Fetches new messages.
@@ -532,6 +528,19 @@ impl Imap {
.context("prefetch")?;
let read_cnt = msgs.len();
let _fetch_msgs_lock_guard = context.fetch_msgs_mutex.lock().await;
if let Some(stop_token) = &self.background_fetch_stop_token {
if stop_token.is_cancelled() {
// This also stops the transport that cancelled the token,
// so one background fetch receives at most `uids_to_prefetch` messages.
return Ok((0, false));
}
if read_cnt > 0 {
// Cancel the background fetch on the other transports,
// so that `background_fetch_any()` can return as soon as messages are received
// and the UI can show a notification.
stop_token.cancel();
}
}
let mut uids_fetch: Vec<u32> = Vec::new();
let mut available_post_msgs: Vec<String> = Vec::new();

56
src/scheduler.rs
View File

@@ -9,7 +9,7 @@ use async_channel::{self as channel, Receiver, Sender};
use futures::future::try_join_all;
use futures_lite::FutureExt;
use tokio::sync::{RwLock, oneshot};
use tokio::task;
use tokio::task::{self, JoinSet};
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
@@ -282,6 +282,60 @@ impl SchedulerState {
scheduler.interrupt_recently_seen(contact_id, timestamp);
}
}
/// Fetches from all transports at once, each on a dedicated connection,
/// with I/O paused so that the scheduler does not connect as well.
///
/// Returns as soon as one transport fetched messages:
/// the others then fetch nothing more and are dropped,
/// so that a caller woken up by a push notification
/// does not wait for a transport that may never answer.
pub(crate) async fn background_fetch_any(&self, context: &Context) -> Result<()> {
let _pause_guard = self.pause(context).await?;
let stop_token = CancellationToken::new();
let mut set = JoinSet::new();
for (transport_id, param) in ConfiguredLoginParam::load_all(context).await? {
let context = context.clone();
let stop_token = stop_token.clone();
set.spawn(async move {
match background_fetch_from_transport(&context, transport_id, param, stop_token)
.await
{
Ok(fetched) => fetched,
Err(err) => {
warn!(context, "Transport {transport_id}: fetch failed: {err:#}.");
false
}
}
});
}
while let Some(fetched) = set.join_next().await {
if let Ok(true) = fetched {
break;
}
}
Ok(())
}
}
async fn background_fetch_from_transport(
context: &Context,
transport_id: u32,
param: ConfiguredLoginParam,
stop_token: CancellationToken,
) -> Result<bool> {
// A single fetch has nothing to interrupt.
let (_, idle_interrupt_receiver) = channel::bounded(1);
let mut connection = Imap::new(context, transport_id, param, idle_interrupt_receiver).await?;
connection.background_fetch_stop_token = Some(stop_token);
let mut session = connection.prepare(context).await?;
let folder = connection.folder.clone();
connection
.fetch_move_delete(context, &mut session, &folder)
.await
}
#[derive(Debug, Default)]