Compare commits

..

1 Commits

Author SHA1 Message Date
link2xt
000bf718fd feat: custom TLS certificate verification 2026-04-05 07:19:39 +02:00
34 changed files with 1154 additions and 353 deletions

View File

@@ -426,6 +426,14 @@ char* dc_get_blobdir (const dc_context_t* context);
* 1=send a copy of outgoing messages to self (default).
* Sending messages to self is needed for a proper multi-account setup,
* however, on the other hand, may lead to unwanted notifications in non-delta clients.
* - `mvbox_move` = 1=detect chat messages,
* move them to the `DeltaChat` folder,
* and watch the `DeltaChat` folder for updates (default),
* 0=do not move chat-messages
* - `only_fetch_mvbox` = 1=Do not fetch messages from folders other than the
* `DeltaChat` folder. Messages will still be fetched from the
* spam folder.
* 0=watch all folders normally (default)
* - `show_emails` = DC_SHOW_EMAILS_OFF (0)=
* show direct replies to chats only,
* DC_SHOW_EMAILS_ACCEPTED_CONTACTS (1)=

View File

@@ -33,12 +33,6 @@ pub struct EnteredLoginParam {
/// Imap server port.
pub imap_port: Option<u16>,
/// IMAP server folder.
///
/// Defaults to "INBOX" if not set.
/// Should not be an empty string.
pub imap_folder: Option<String>,
/// Imap socket security.
pub imap_security: Option<Socket>,
@@ -91,7 +85,6 @@ impl From<dc::EnteredLoginParam> for EnteredLoginParam {
password: param.imap.password,
imap_server: param.imap.server.into_option(),
imap_port: param.imap.port.into_option(),
imap_folder: param.imap.folder.into_option(),
imap_security: imap_security.into_option(),
imap_user: param.imap.user.into_option(),
smtp_server: param.smtp.server.into_option(),
@@ -111,15 +104,14 @@ impl TryFrom<EnteredLoginParam> for dc::EnteredLoginParam {
fn try_from(param: EnteredLoginParam) -> Result<Self> {
Ok(Self {
addr: param.addr,
imap: dc::EnteredImapLoginParam {
imap: dc::EnteredServerLoginParam {
server: param.imap_server.unwrap_or_default(),
port: param.imap_port.unwrap_or_default(),
folder: param.imap_folder.unwrap_or_default(),
security: param.imap_security.unwrap_or_default().into(),
user: param.imap_user.unwrap_or_default(),
password: param.password,
},
smtp: dc::EnteredSmtpLoginParam {
smtp: dc::EnteredServerLoginParam {
server: param.smtp_server.unwrap_or_default(),
port: param.smtp_port.unwrap_or_default(),
security: param.smtp_security.unwrap_or_default().into(),

View File

@@ -2,13 +2,32 @@ import logging
import re
import time
import pytest
from imap_tools import AND, U
from deltachat_rpc_client import Contact, EventType, Message
def test_move_works(acfactory, direct_imap):
ac1, ac2 = acfactory.get_online_accounts(2)
ac2_direct_imap = direct_imap(ac2)
ac2_direct_imap.create_folder("DeltaChat")
ac2.set_config("mvbox_move", "1")
ac2.bring_online()
chat = ac1.create_chat(ac2)
chat.send_text("message1")
# Message is moved to the movebox
ac2.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
# Message is downloaded
msg = ac2.wait_for_incoming_msg().get_snapshot()
assert msg.text == "message1"
def test_reactions_for_a_reordering_move(acfactory, direct_imap):
"""When a batch of messages is moved from Inbox to another folder with a single MOVE command,
"""When a batch of messages is moved from Inbox to DeltaChat folder with a single MOVE command,
their UIDs may be reordered (e.g. Gmail is known for that) which led to that messages were
processed by receive_imf in the wrong order, and, particularly, reactions were processed before
messages they refer to and thus dropped.
@@ -18,6 +37,9 @@ def test_reactions_for_a_reordering_move(acfactory, direct_imap):
addr, password = acfactory.get_credentials()
ac2 = acfactory.get_unconfigured_account()
ac2.add_or_update_transport({"addr": addr, "password": password})
ac2_direct_imap = direct_imap(ac2)
ac2_direct_imap.create_folder("DeltaChat")
ac2.set_config("mvbox_move", "1")
assert ac2.is_configured()
ac2.bring_online()
@@ -33,17 +55,11 @@ def test_reactions_for_a_reordering_move(acfactory, direct_imap):
react_str = "\N{THUMBS UP SIGN}"
msg1.send_reaction(react_str).wait_until_delivered()
logging.info("moving messages to ac2's movebox folder in the reverse order")
logging.info("moving messages to ac2's DeltaChat folder in the reverse order")
ac2_direct_imap = direct_imap(ac2)
ac2_direct_imap.create_folder("Movebox")
ac2_direct_imap.connect()
for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()], reverse=True):
ac2_direct_imap.conn.move(uid, "Movebox")
logging.info("moving messages back")
ac2_direct_imap.select_folder("Movebox")
for uid in sorted([m.uid for m in ac2_direct_imap.get_all_messages()]):
ac2_direct_imap.conn.move(uid, "INBOX")
ac2_direct_imap.conn.move(uid, "DeltaChat")
logging.info("receiving messages by ac2")
ac2.start_io()
@@ -56,22 +72,33 @@ def test_reactions_for_a_reordering_move(acfactory, direct_imap):
assert list(reactions.reactions_by_contact.values())[0] == [react_str]
def test_moved_markseen(acfactory, direct_imap, log):
def test_move_works_on_self_sent(acfactory, direct_imap):
ac1, ac2 = acfactory.get_online_accounts(2)
# Create and enable movebox.
ac1_direct_imap = direct_imap(ac1)
ac1_direct_imap.create_folder("DeltaChat")
ac1.set_config("mvbox_move", "1")
ac1.set_config("bcc_self", "1")
ac1.bring_online()
chat = ac1.create_chat(ac2)
chat.send_text("message1")
ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
chat.send_text("message2")
ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
chat.send_text("message3")
ac1.wait_for_event(EventType.IMAP_MESSAGE_MOVED)
def test_moved_markseen(acfactory, direct_imap):
"""Test that message already moved to DeltaChat folder is marked as seen."""
ac1 = acfactory.get_online_account()
addr, password = acfactory.get_credentials()
ac2 = acfactory.get_unconfigured_account()
ac2.add_or_update_transport({"addr": addr, "password": password})
ac2.bring_online()
log.section("ac2: creating DeltaChat folder")
ac1, ac2 = acfactory.get_online_accounts(2)
ac2_direct_imap = direct_imap(ac2)
ac2_direct_imap.create_folder("DeltaChat")
ac2.set_config("mvbox_move", "1")
ac2.set_config("delete_server_after", "0")
ac2.set_config("sync_msgs", "0") # Do not send a sync message when accepting a contact request.
ac2.add_or_update_transport({"addr": addr, "password": password, "imapFolder": "DeltaChat"})
ac2.bring_online()
ac2.stop_io()
@@ -81,7 +108,6 @@ def test_moved_markseen(acfactory, direct_imap, log):
idle2.wait_for_new_message()
# Emulate moving of the message to DeltaChat folder by Sieve rule.
log.section("ac2: moving message into DeltaChat folder")
ac2_direct_imap.conn.move(["*"], "DeltaChat")
ac2_direct_imap.select_folder("DeltaChat")
assert len(list(ac2_direct_imap.conn.fetch("*", mark_seen=False))) == 1
@@ -105,11 +131,17 @@ def test_moved_markseen(acfactory, direct_imap, log):
assert len(list(ac2_direct_imap.conn.fetch(AND(seen=True, uid=U(1, "*")), mark_seen=False))) == 1
def test_markseen_message_and_mdn(acfactory, direct_imap):
@pytest.mark.parametrize("mvbox_move", [True, False])
def test_markseen_message_and_mdn(acfactory, direct_imap, mvbox_move):
ac1, ac2 = acfactory.get_online_accounts(2)
for ac in ac1, ac2:
ac.set_config("delete_server_after", "0")
if mvbox_move:
ac_direct_imap = direct_imap(ac)
ac_direct_imap.create_folder("DeltaChat")
ac.set_config("mvbox_move", "1")
ac.bring_online()
# Do not send BCC to self, we only want to test MDN on ac1.
ac1.set_config("bcc_self", "0")
@@ -118,7 +150,10 @@ def test_markseen_message_and_mdn(acfactory, direct_imap):
msg = ac2.wait_for_incoming_msg()
msg.mark_seen()
rex = re.compile("Marked messages [0-9]+ in folder INBOX as seen.")
if mvbox_move:
rex = re.compile("Marked messages [0-9]+ in folder DeltaChat as seen.")
else:
rex = re.compile("Marked messages [0-9]+ in folder INBOX as seen.")
for ac in ac1, ac2:
while True:
@@ -126,11 +161,12 @@ def test_markseen_message_and_mdn(acfactory, direct_imap):
if event.kind == EventType.INFO and rex.search(event.msg):
break
folder = "mvbox" if mvbox_move else "inbox"
ac1_direct_imap = direct_imap(ac1)
ac2_direct_imap = direct_imap(ac2)
ac1_direct_imap.select_folder("INBOX")
ac2_direct_imap.select_folder("INBOX")
ac1_direct_imap.select_config_folder(folder)
ac2_direct_imap.select_config_folder(folder)
# Check that the mdn is marked as seen
assert len(list(ac1_direct_imap.conn.fetch(AND(seen=True), mark_seen=False))) == 1

View File

@@ -9,6 +9,10 @@ def test_add_second_address(acfactory) -> None:
account = acfactory.new_configured_account()
assert len(account.list_transports()) == 1
# When the first transport is created,
# mvbox_move and only_fetch_mvbox should be disabled.
assert account.get_config("mvbox_move") == "0"
assert account.get_config("only_fetch_mvbox") == "0"
assert account.get_config("show_emails") == "2"
qr = acfactory.get_account_qr()
@@ -28,10 +32,32 @@ def test_add_second_address(acfactory) -> None:
account.delete_transport(second_addr)
assert len(account.list_transports()) == 2
# Enabling mvbox_move or only_fetch_mvbox
# is not allowed when multi-transport is enabled.
for option in ["mvbox_move", "only_fetch_mvbox"]:
with pytest.raises(JsonRpcError):
account.set_config(option, "1")
# show_emails does not matter for multi-relay, can be set to anything
account.set_config("show_emails", "0")
@pytest.mark.parametrize("key", ["mvbox_move", "only_fetch_mvbox"])
def test_no_second_transport_with_mvbox(acfactory, key) -> None:
"""Test that second transport cannot be configured if mvbox is used."""
account = acfactory.new_configured_account()
assert len(account.list_transports()) == 1
assert account.get_config("mvbox_move") == "0"
assert account.get_config("only_fetch_mvbox") == "0"
qr = acfactory.get_account_qr()
account.set_config(key, "1")
with pytest.raises(JsonRpcError):
account.add_transport_from_qr(qr)
def test_second_transport_without_classic_emails(acfactory) -> None:
"""Test that second transport can be configured if classic emails are not fetched."""
account = acfactory.new_configured_account()
@@ -121,13 +147,44 @@ def test_download_on_demand(acfactory) -> None:
assert msg.get_snapshot().download_state == dstate
@pytest.mark.parametrize("is_chatmail", ["0", "1"])
def test_mvbox_move_first_transport(acfactory, is_chatmail) -> None:
"""Test that mvbox_move is disabled by default even for non-chatmail accounts.
Disabling mvbox_move is required to be able to setup a second transport.
"""
account = acfactory.get_unconfigured_account()
account.set_config("fix_is_chatmail", "1")
account.set_config("is_chatmail", is_chatmail)
# The default value when the setting is unset is "1".
# This is not changed for compatibility with old databases
# imported from backups.
assert account.get_config("mvbox_move") == "1"
qr = acfactory.get_account_qr()
account.add_transport_from_qr(qr)
# Once the first transport is set up,
# mvbox_move is disabled.
assert account.get_config("mvbox_move") == "0"
assert account.get_config("is_chatmail") == is_chatmail
def test_reconfigure_transport(acfactory) -> None:
"""Test that reconfiguring the transport works."""
"""Test that reconfiguring the transport works
even if settings not supported for multi-transport
like mvbox_move are enabled."""
account = acfactory.get_online_account()
account.set_config("mvbox_move", "1")
[transport] = account.list_transports()
account.add_or_update_transport(transport)
# Reconfiguring the transport should not reset
# the settings as if when configuring the first transport.
assert account.get_config("mvbox_move") == "1"
def test_transport_synchronization(acfactory, log) -> None:
"""Test synchronization of transports between devices."""

View File

@@ -522,6 +522,7 @@ class ACFactory:
ac = self.get_unconfigured_account()
assert "addr" in configdict and "mail_pw" in configdict, configdict
configdict.setdefault("bcc_self", False)
configdict.setdefault("mvbox_move", False)
configdict.setdefault("sync_msgs", False)
configdict.setdefault("delete_server_after", 0)
ac.update_config(configdict)

View File

@@ -52,19 +52,19 @@ class TestOfflineAccountBasic:
def test_set_config_int_conversion(self, acfactory):
ac1 = acfactory.get_unconfigured_account()
ac1.set_config("bcc_self", False)
assert ac1.get_config("bcc_self") == "0"
ac1.set_config("bcc_self", True)
assert ac1.get_config("bcc_self") == "1"
ac1.set_config("bcc_self", 0)
assert ac1.get_config("bcc_self") == "0"
ac1.set_config("bcc_self", 1)
assert ac1.get_config("bcc_self") == "1"
ac1.set_config("mvbox_move", False)
assert ac1.get_config("mvbox_move") == "0"
ac1.set_config("mvbox_move", True)
assert ac1.get_config("mvbox_move") == "1"
ac1.set_config("mvbox_move", 0)
assert ac1.get_config("mvbox_move") == "0"
ac1.set_config("mvbox_move", 1)
assert ac1.get_config("mvbox_move") == "1"
def test_update_config(self, acfactory):
ac1 = acfactory.get_unconfigured_account()
ac1.update_config({"bcc_self": True})
assert ac1.get_config("bcc_self") == "1"
ac1.update_config({"mvbox_move": False})
assert ac1.get_config("mvbox_move") == "0"
def test_has_bccself(self, acfactory):
ac1 = acfactory.get_unconfigured_account()

View File

@@ -155,6 +155,18 @@ pub enum Config {
#[strum(props(default = "1"))]
MdnsEnabled,
/// True if chat messages should be moved to a separate folder. Auto-sent messages like sync
/// ones are moved there anyway.
#[strum(props(default = "1"))]
MvboxMove,
/// Watch for new messages in the "Mvbox" (aka DeltaChat folder) only.
///
/// This will not entirely disable other folders, e.g. the spam folder will also still
/// be watched for new messages.
#[strum(props(default = "0"))]
OnlyFetchMvbox,
/// Whether to show classic emails or only chat messages.
#[strum(props(default = "2"))] // also change ShowEmails.default() on changes
ShowEmails,
@@ -256,6 +268,9 @@ pub enum Config {
/// Configured folder for incoming messages.
ConfiguredInboxFolder,
/// Configured folder for chat messages.
ConfiguredMvboxFolder,
/// Unix timestamp of the last successful configuration.
ConfiguredTimestamp,
@@ -452,6 +467,7 @@ impl Config {
self,
Self::Displayname
| Self::MdnsEnabled
| Self::MvboxMove
| Self::ShowEmails
| Self::Selfavatar
| Self::Selfstatus,
@@ -460,7 +476,10 @@ impl Config {
/// Whether the config option needs an IO scheduler restart to take effect.
pub(crate) fn needs_io_restart(&self) -> bool {
matches!(self, Config::ConfiguredAddr)
matches!(
self,
Config::MvboxMove | Config::OnlyFetchMvbox | Config::ConfiguredAddr
)
}
}
@@ -575,6 +594,13 @@ impl Context {
.is_some_and(|x| x != 0))
}
/// Returns true if movebox ("DeltaChat" folder) should be watched.
pub(crate) async fn should_watch_mvbox(&self) -> Result<bool> {
Ok(self.get_config_bool(Config::MvboxMove).await?
|| self.get_config_bool(Config::OnlyFetchMvbox).await?
|| !self.get_config_bool(Config::IsChatmail).await?)
}
/// Returns true if sync messages should be sent.
pub(crate) async fn should_send_sync_msgs(&self) -> Result<bool> {
Ok(self.get_config_bool(Config::SyncMsgs).await?
@@ -656,6 +682,8 @@ impl Context {
| Config::ProxyEnabled
| Config::BccSelf
| Config::MdnsEnabled
| Config::MvboxMove
| Config::OnlyFetchMvbox
| Config::Configured
| Config::Bot
| Config::NotifyAboutWrongPw
@@ -678,6 +706,11 @@ impl Context {
pub async fn set_config(&self, key: Config, value: Option<&str>) -> Result<()> {
Self::check_config(key, value)?;
let n_transports = self.count_transports().await?;
if n_transports > 1 && matches!(key, Config::MvboxMove | Config::OnlyFetchMvbox) {
bail!("Cannot reconfigure {key} when multiple transports are configured");
}
let _pause = match key.needs_io_restart() {
true => self.scheduler.pause(self).await?,
_ => Default::default(),
@@ -756,6 +789,12 @@ impl Context {
.set_raw_config(key.as_ref(), value.map(|s| s.to_lowercase()).as_deref())
.await?;
}
Config::MvboxMove => {
self.sql.set_raw_config(key.as_ref(), value).await?;
self.sql
.set_raw_config(constants::DC_FOLDERS_CONFIGURED_KEY, None)
.await?;
}
Config::ConfiguredAddr => {
let Some(addr) = value else {
bail!("Cannot unset configured_addr");

View File

@@ -196,11 +196,11 @@ async fn test_sync() -> Result<()> {
sync(&alice0, &alice1).await;
assert_eq!(alice1.get_config_bool(Config::MdnsEnabled).await?, false);
{
let val = alice0.get_config_bool(Config::ShowEmails).await?;
alice0.set_config_bool(Config::ShowEmails, !val).await?;
for key in [Config::ShowEmails, Config::MvboxMove] {
let val = alice0.get_config_bool(key).await?;
alice0.set_config_bool(key, !val).await?;
sync(&alice0, &alice1).await;
assert_eq!(alice1.get_config_bool(Config::ShowEmails).await?, !val);
assert_eq!(alice1.get_config_bool(key).await?, !val);
}
// `Config::SyncMsgs` mustn't be synced.

View File

@@ -315,16 +315,31 @@ impl Context {
(&param.addr,),
)
.await?
&& self
{
// Should be checked before `MvboxMove` because the latter makes no sense in presense of
// `OnlyFetchMvbox` and even grayed out in the UIs in this case.
if self.get_config(Config::OnlyFetchMvbox).await?.as_deref() != Some("0") {
bail!(
"To use additional relays, disable the legacy option \"Settings / Advanced / Only Fetch from DeltaChat Folder\"."
);
}
if self.get_config(Config::MvboxMove).await?.as_deref() != Some("0") {
bail!(
"To use additional relays, disable the legacy option \"Settings / Advanced / Move automatically to DeltaChat Folder\"."
);
}
if self
.sql
.count("SELECT COUNT(*) FROM transports", ())
.await?
>= MAX_TRANSPORT_RELAYS
{
bail!(
"You have reached the maximum number of relays ({}).",
MAX_TRANSPORT_RELAYS
)
{
bail!(
"You have reached the maximum number of relays ({}).",
MAX_TRANSPORT_RELAYS
)
}
}
let provider = match configure(self, param).await {
@@ -537,7 +552,6 @@ async fn get_configured_param(
.collect(),
imap_user: param.imap.user.clone(),
imap_password: param.imap.password.clone(),
imap_folder: Some(param.imap.folder.clone()).filter(|folder| !folder.is_empty()),
smtp: servers
.iter()
.filter_map(|params| {
@@ -630,6 +644,10 @@ async fn configure(ctx: &Context, param: &EnteredLoginParam) -> Result<Option<&'
progress!(ctx, 900);
let is_configured = ctx.is_configured().await?;
if !is_configured {
ctx.sql.set_raw_config("mvbox_move", Some("0")).await?;
ctx.sql.set_raw_config("only_fetch_mvbox", None).await?;
}
if !ctx.get_config_bool(Config::FixIsChatmail).await? {
if imap_session.is_chatmail() {
ctx.sql.set_raw_config("is_chatmail", Some("1")).await?;
@@ -793,7 +811,7 @@ pub enum Error {
mod tests {
use super::*;
use crate::config::Config;
use crate::login_param::EnteredImapLoginParam;
use crate::login_param::EnteredServerLoginParam;
use crate::test_utils::TestContext;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -812,7 +830,7 @@ mod tests {
let entered_param = EnteredLoginParam {
addr: "alice@example.org".to_string(),
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
user: "alice@example.net".to_string(),
password: "foobar".to_string(),
..Default::default()

View File

@@ -210,6 +210,11 @@ pub const WORSE_IMAGE_SIZE: u32 = 640;
/// usage by UIs.
pub const MAX_RCVD_IMAGE_PIXELS: u32 = 50_000_000;
// Key for the folder configuration version (see below).
pub(crate) const DC_FOLDERS_CONFIGURED_KEY: &str = "folders_configured";
// this value can be increased if the folder configuration is changed and must be redone on next program start
pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 5;
// If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
// chunks. This does not affect MIME's `To:` header. Can be overwritten by setting
// `max_smtp_rcpt_to` in the provider db.

View File

@@ -20,16 +20,16 @@ use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSI
use crate::contact::{Contact, ContactId};
use crate::debug_logging::DebugLogging;
use crate::events::{Event, EventEmitter, EventType, Events};
use crate::imap::{Imap, ServerMetadata};
use crate::imap::{FolderMeaning, Imap, ServerMetadata};
use crate::key::self_fingerprint;
use crate::log::warn;
use crate::logged_debug_assert;
use crate::message::{self, MessageState, MsgId};
use crate::net::tls::TlsSessionStore;
use crate::net::tls::{SpkiHashStore, TlsSessionStore};
use crate::peer_channels::Iroh;
use crate::push::PushSubscriber;
use crate::quota::QuotaInfo;
use crate::scheduler::{ConnectivityStore, SchedulerState};
use crate::scheduler::{ConnectivityStore, SchedulerState, convert_folder_meaning};
use crate::sql::Sql;
use crate::stock_str::StockStrings;
use crate::timesmearing::SmearedTimestamp;
@@ -308,6 +308,13 @@ pub struct InnerContext {
/// TLS session resumption cache.
pub(crate) tls_session_store: TlsSessionStore,
/// Store for TLS SPKI hashes.
///
/// Used to remember public keys
/// of TLS certificates to accept them
/// even after they expire.
pub(crate) spki_hash_store: SpkiHashStore,
/// Iroh for realtime peer channels.
pub(crate) iroh: Arc<RwLock<Option<Iroh>>>,
@@ -511,6 +518,7 @@ impl Context {
push_subscriber,
push_subscribed: AtomicBool::new(false),
tls_session_store: TlsSessionStore::new(),
spki_hash_store: SpkiHashStore::new(),
iroh: Arc::new(RwLock::new(None)),
self_fingerprint: OnceLock::new(),
self_public_key: Mutex::new(None),
@@ -623,10 +631,17 @@ impl Context {
let mut session = connection.prepare(self).await?;
// Fetch IMAP folders.
let folder = connection.folder.clone();
connection
.fetch_move_delete(self, &mut session, &folder)
.await?;
// Inbox is fetched before Mvbox because fetching from Inbox
// may result in moving some messages to Mvbox.
for folder_meaning in [FolderMeaning::Inbox, FolderMeaning::Mvbox] {
if let Some((_folder_config, watch_folder)) =
convert_folder_meaning(self, folder_meaning).await?
{
connection
.fetch_move_delete(self, &mut session, &watch_folder, folder_meaning)
.await?;
}
}
// Update quota (to send warning if full) - but only check it once in a while.
// note: For now this only checks quota of primary transport,
@@ -637,7 +652,7 @@ impl Context {
DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT,
)
.await
&& let Err(err) = self.update_recent_quota(&mut session, &folder).await
&& let Err(err) = self.update_recent_quota(&mut session).await
{
warn!(self, "Failed to update quota: {err:#}.");
}
@@ -877,6 +892,23 @@ impl Context {
Err(err) => format!("<key failure: {err}>"),
};
let mvbox_move = self.get_config_int(Config::MvboxMove).await?;
let only_fetch_mvbox = self.get_config_int(Config::OnlyFetchMvbox).await?;
let folders_configured = self
.sql
.get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
.await?
.unwrap_or_default();
let configured_inbox_folder = self
.get_config(Config::ConfiguredInboxFolder)
.await?
.unwrap_or_else(|| "<unset>".to_string());
let configured_mvbox_folder = self
.get_config(Config::ConfiguredMvboxFolder)
.await?
.unwrap_or_else(|| "<unset>".to_string());
let mut res = get_info();
// insert values
@@ -952,6 +984,14 @@ impl Context {
.await?
.to_string(),
);
res.insert("mvbox_move", mvbox_move.to_string());
res.insert("only_fetch_mvbox", only_fetch_mvbox.to_string());
res.insert(
constants::DC_FOLDERS_CONFIGURED_KEY,
folders_configured.to_string(),
);
res.insert("configured_inbox_folder", configured_inbox_folder);
res.insert("configured_mvbox_folder", configured_mvbox_folder);
res.insert("mdns_enabled", mdns_enabled.to_string());
res.insert("bcc_self", bcc_self.to_string());
res.insert("sync_msgs", sync_msgs.to_string());
@@ -1251,6 +1291,12 @@ ORDER BY m.timestamp DESC,m.id DESC",
Ok(list)
}
/// Returns true if given folder name is the name of the "DeltaChat" folder.
pub async fn is_mvbox(&self, folder_name: &str) -> Result<bool> {
let mvbox = self.get_config(Config::ConfiguredMvboxFolder).await?;
Ok(mvbox.as_deref() == Some(folder_name))
}
pub(crate) fn derive_blobdir(dbfile: &Path) -> PathBuf {
let mut blob_fname = OsString::new();
blob_fname.push(dbfile.file_name().unwrap_or_default());

View File

@@ -27,14 +27,13 @@ use crate::calls::{
use crate::chat::{self, ChatId, ChatIdBlocked, add_device_msg};
use crate::chatlist_events;
use crate::config::Config;
use crate::constants::{Blocked, DC_VERSION_STR};
use crate::constants::{self, Blocked, DC_VERSION_STR};
use crate::contact::ContactId;
use crate::context::Context;
use crate::ensure_and_debug_assert;
use crate::events::EventType;
use crate::headerdef::{HeaderDef, HeaderDefMap};
use crate::log::{LogExt, warn};
use crate::message::{self, Message, MessageState, MsgId};
use crate::message::{self, Message, MessageState, MessengerMessage, MsgId};
use crate::mimeparser;
use crate::net::proxy::ProxyConfig;
use crate::net::session::SessionStream;
@@ -92,9 +91,6 @@ pub(crate) struct Imap {
oauth2: bool,
/// Watched folder.
pub(crate) folder: String,
authentication_failed_once: bool,
pub(crate) connectivity: ConnectivityStore,
@@ -166,6 +162,7 @@ pub enum FolderMeaning {
/// Spam folder.
Spam,
Inbox,
Mvbox,
Trash,
/// Virtual folders.
@@ -177,6 +174,19 @@ pub enum FolderMeaning {
Virtual,
}
impl FolderMeaning {
pub fn to_config(self) -> Option<Config> {
match self {
FolderMeaning::Unknown => None,
FolderMeaning::Spam => None,
FolderMeaning::Inbox => Some(Config::ConfiguredInboxFolder),
FolderMeaning::Mvbox => Some(Config::ConfiguredMvboxFolder),
FolderMeaning::Trash => None,
FolderMeaning::Virtual => None,
}
}
}
struct UidGrouper<T: Iterator<Item = (i64, u32, String)>> {
inner: Peekable<T>,
}
@@ -253,11 +263,6 @@ impl Imap {
let addr = &param.addr;
let strict_tls = param.strict_tls(proxy_config.is_some());
let oauth2 = param.oauth2;
let folder = param
.imap_folder
.clone()
.unwrap_or_else(|| "INBOX".to_string());
ensure_and_debug_assert!(!folder.is_empty(), "Watched folder name cannot be empty");
let (resync_request_sender, resync_request_receiver) = async_channel::bounded(1);
Ok(Imap {
transport_id,
@@ -268,7 +273,6 @@ impl Imap {
proxy_config,
strict_tls,
oauth2,
folder,
authentication_failed_once: false,
connectivity: Default::default(),
conn_last_try: UNIX_EPOCH,
@@ -486,7 +490,7 @@ impl Imap {
/// that folders are created and IMAP capabilities are determined.
pub(crate) async fn prepare(&mut self, context: &Context) -> Result<Session> {
let configuring = false;
let session = match self.connect(context, configuring).await {
let mut session = match self.connect(context, configuring).await {
Ok(session) => session,
Err(err) => {
self.connectivity.set_err(context, &err);
@@ -494,6 +498,14 @@ impl Imap {
}
};
let folders_configured = context
.sql
.get_raw_config_int(constants::DC_FOLDERS_CONFIGURED_KEY)
.await?;
if folders_configured.unwrap_or_default() < constants::DC_FOLDERS_CONFIGURED_VERSION {
self.configure_folders(context, &mut session).await?;
}
Ok(session)
}
@@ -506,15 +518,15 @@ impl Imap {
context: &Context,
session: &mut Session,
watch_folder: &str,
folder_meaning: FolderMeaning,
) -> Result<()> {
ensure_and_debug_assert!(!watch_folder.is_empty(), "Watched folder cannot be empty");
if !context.sql.is_open().await {
// probably shutdown
bail!("IMAP operation attempted while it is torn down");
}
let msgs_fetched = self
.fetch_new_messages(context, session, watch_folder)
.fetch_new_messages(context, session, watch_folder, folder_meaning)
.await
.context("fetch_new_messages")?;
if msgs_fetched && context.get_config_delete_device_after().await?.is_some() {
@@ -542,9 +554,19 @@ impl Imap {
context: &Context,
session: &mut Session,
folder: &str,
folder_meaning: FolderMeaning,
) -> Result<bool> {
let transport_id = session.transport_id();
if should_ignore_folder(context, folder, folder_meaning).await? {
info!(
context,
"Transport {transport_id}: Not fetching from {folder:?}."
);
session.new_mail = false;
return Ok(false);
}
let folder_exists = session
.select_with_uidvalidity(context, folder)
.await
@@ -567,7 +589,9 @@ impl Imap {
let mut read_cnt = 0;
loop {
let (n, fetch_more) = self.fetch_new_msg_batch(context, session, folder).await?;
let (n, fetch_more) = self
.fetch_new_msg_batch(context, session, folder, folder_meaning)
.await?;
read_cnt += n;
if !fetch_more {
return Ok(read_cnt > 0);
@@ -582,6 +606,7 @@ impl Imap {
context: &Context,
session: &mut Session,
folder: &str,
folder_meaning: FolderMeaning,
) -> Result<(usize, bool)> {
let transport_id = self.transport_id;
let uid_validity = get_uidvalidity(context, transport_id, folder).await?;
@@ -651,7 +676,13 @@ impl Imap {
info!(context, "Deleting locally deleted message {message_id}.");
}
let target = if delete { "" } else { folder };
let _target;
let target = if delete {
""
} else {
_target = target_folder(context, folder, folder_meaning, &headers).await?;
&_target
};
context
.sql
@@ -679,9 +710,18 @@ impl Imap {
// message, move it to the movebox and then download the second message before
// downloading the first one, if downloading from inbox before moving is allowed.
if folder == target
&& prefetch_should_download(context, &headers, &message_id, fetch_response.flags())
.await
.context("prefetch_should_download")?
// Never download messages directly from the spam folder.
// If the sender is known, the message will be moved to the Inbox or Mvbox
// and then we download the message from there.
// Also see `spam_target_folder_cfg()`.
&& folder_meaning != FolderMeaning::Spam
&& prefetch_should_download(
context,
&headers,
&message_id,
fetch_response.flags(),
)
.await.context("prefetch_should_download")?
{
if headers
.get_header_value(HeaderDef::ChatIsPostMessage)
@@ -1622,8 +1662,13 @@ impl Session {
// Store new encrypted device token on the server
// even if it is the same as the old one.
if let Some(encrypted_device_token) = new_encrypted_device_token {
let folder = context
.get_config(Config::ConfiguredInboxFolder)
.await?
.context("INBOX is not configured")?;
self.run_command_and_check_ok(&format_setmetadata(
"INBOX",
&folder,
&encrypted_device_token,
))
.await
@@ -1668,6 +1713,117 @@ impl Session {
}
Ok(())
}
/// Attempts to configure mvbox.
///
/// Tries to find any folder examining `folders` in the order they go.
/// This method does not use LIST command to ensure that
/// configuration works even if mailbox lookup is forbidden via Access Control List (see
/// <https://datatracker.ietf.org/doc/html/rfc4314>).
///
/// Returns first found folder name.
async fn configure_mvbox<'a>(
&mut self,
context: &Context,
folders: &[&'a str],
) -> Result<Option<&'a str>> {
// Close currently selected folder if needed.
// We are going to select folders using low-level EXAMINE operations below.
self.maybe_close_folder(context).await?;
for folder in folders {
info!(context, "Looking for MVBOX-folder \"{}\"...", &folder);
let res = self.examine(&folder).await;
if res.is_ok() {
info!(
context,
"MVBOX-folder {:?} successfully selected, using it.", &folder
);
self.close().await?;
// Before moving emails to the mvbox we need to remember its UIDVALIDITY, otherwise
// emails moved before that wouldn't be fetched but considered "old" instead.
let folder_exists = self.select_with_uidvalidity(context, folder).await?;
ensure!(folder_exists, "No MVBOX folder {:?}??", &folder);
return Ok(Some(folder));
}
}
Ok(None)
}
}
impl Imap {
pub(crate) async fn configure_folders(
&mut self,
context: &Context,
session: &mut Session,
) -> Result<()> {
let mut folders = session
.list(Some(""), Some("*"))
.await
.context("list_folders failed")?;
let mut delimiter = ".".to_string();
let mut delimiter_is_default = true;
let mut folder_configs = BTreeMap::new();
while let Some(folder) = folders.try_next().await? {
info!(context, "Scanning folder: {:?}", folder);
// Update the delimiter iff there is a different one, but only once.
if let Some(d) = folder.delimiter()
&& delimiter_is_default
&& !d.is_empty()
&& delimiter != d
{
delimiter = d.to_string();
delimiter_is_default = false;
}
let folder_meaning = get_folder_meaning_by_attrs(folder.attributes());
let folder_name_meaning = get_folder_meaning_by_name(folder.name());
if let Some(config) = folder_meaning.to_config() {
// Always takes precedence
folder_configs.insert(config, folder.name().to_string());
} else if let Some(config) = folder_name_meaning.to_config() {
// only set if none has been already set
folder_configs
.entry(config)
.or_insert_with(|| folder.name().to_string());
}
}
drop(folders);
info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
let fallback_folder = format!("INBOX{delimiter}DeltaChat");
let mvbox_folder = session
.configure_mvbox(context, &["DeltaChat", &fallback_folder])
.await
.context("failed to configure mvbox")?;
context
.set_config_internal(Config::ConfiguredInboxFolder, Some("INBOX"))
.await?;
if let Some(mvbox_folder) = mvbox_folder {
info!(context, "Setting MVBOX FOLDER TO {}", &mvbox_folder);
context
.set_config_internal(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
.await?;
}
for (config, name) in folder_configs {
context.set_config_internal(config, Some(&name)).await?;
}
context
.sql
.set_raw_config_int(
constants::DC_FOLDERS_CONFIGURED_KEY,
constants::DC_FOLDERS_CONFIGURED_VERSION,
)
.await?;
info!(context, "FINISHED configuring IMAP-folders.");
Ok(())
}
}
impl Session {
@@ -1801,7 +1957,15 @@ async fn spam_target_folder_cfg(
return Ok(None);
}
Ok(Some(Config::ConfiguredInboxFolder))
if needs_move_to_mvbox(context, headers).await?
// If OnlyFetchMvbox is set, we don't want to move the message to
// the inbox where we wouldn't fetch it again:
|| context.get_config_bool(Config::OnlyFetchMvbox).await?
{
Ok(Some(Config::ConfiguredMvboxFolder))
} else {
Ok(Some(Config::ConfiguredInboxFolder))
}
}
/// Returns `ConfiguredInboxFolder` or `ConfiguredMvboxFolder` if
@@ -1812,12 +1976,16 @@ pub async fn target_folder_cfg(
folder_meaning: FolderMeaning,
headers: &[mailparse::MailHeader<'_>],
) -> Result<Option<Config>> {
if folder == "DeltaChat" {
if context.is_mvbox(folder).await? {
return Ok(None);
}
if folder_meaning == FolderMeaning::Spam {
spam_target_folder_cfg(context, headers).await
} else if folder_meaning == FolderMeaning::Inbox
&& needs_move_to_mvbox(context, headers).await?
{
Ok(Some(Config::ConfiguredMvboxFolder))
} else {
Ok(None)
}
@@ -1838,6 +2006,27 @@ pub async fn target_folder(
}
}
async fn needs_move_to_mvbox(
context: &Context,
headers: &[mailparse::MailHeader<'_>],
) -> Result<bool> {
let has_chat_version = headers.get_header_value(HeaderDef::ChatVersion).is_some();
if !context.get_config_bool(Config::MvboxMove).await? {
return Ok(false);
}
if has_chat_version {
Ok(true)
} else if let Some(parent) = get_prefetch_parent_message(context, headers).await? {
match parent.is_dc_message {
MessengerMessage::No => Ok(false),
MessengerMessage::Yes | MessengerMessage::Reply => Ok(true),
}
} else {
Ok(false)
}
}
/// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
// TODO: lots languages missing - maybe there is a list somewhere on other MUAs?
// however, if we fail to find out the sent-folder,
@@ -2168,6 +2357,21 @@ async fn get_modseq(context: &Context, transport_id: u32, folder: &str) -> Resul
.unwrap_or(0))
}
/// Whether to ignore fetching messages from a folder.
///
/// This caters for the [`Config::OnlyFetchMvbox`] setting which means mails from folders
/// not explicitly watched should not be fetched.
async fn should_ignore_folder(
context: &Context,
folder: &str,
folder_meaning: FolderMeaning,
) -> Result<bool> {
if !context.get_config_bool(Config::OnlyFetchMvbox).await? {
return Ok(false);
}
Ok(!(context.is_mvbox(folder).await? || folder_meaning == FolderMeaning::Spam))
}
/// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
/// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
@@ -2227,5 +2431,23 @@ impl std::fmt::Display for UidRange {
}
}
pub(crate) async fn get_watched_folder_configs(context: &Context) -> Result<Vec<Config>> {
let mut res = vec![Config::ConfiguredInboxFolder];
if context.should_watch_mvbox().await? {
res.push(Config::ConfiguredMvboxFolder);
}
Ok(res)
}
pub(crate) async fn get_watched_folders(context: &Context) -> Result<Vec<String>> {
let mut res = Vec::new();
for folder_config in get_watched_folder_configs(context).await? {
if let Some(folder) = context.get_config(folder_config).await? {
res.push(folder);
}
}
Ok(res)
}
#[cfg(test)]
mod imap_tests;

View File

@@ -220,6 +220,8 @@ impl Client {
alpn(addr.port()),
logging_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
let buffered_stream = BufWriter::new(tls_stream);
@@ -282,6 +284,8 @@ impl Client {
"",
tcp_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await
.context("STARTTLS upgrade failed")?;
@@ -310,6 +314,8 @@ impl Client {
alpn(port),
proxy_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
let buffered_stream = BufWriter::new(tls_stream);
@@ -373,6 +379,8 @@ impl Client {
"",
proxy_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await
.context("STARTTLS upgrade failed")?;

View File

@@ -120,7 +120,11 @@ impl Session {
impl Imap {
/// Idle using polling.
pub(crate) async fn fake_idle(&mut self, context: &Context, watch_folder: &str) -> Result<()> {
pub(crate) async fn fake_idle(
&mut self,
context: &Context,
watch_folder: String,
) -> Result<()> {
let fake_idle_start_time = tools::Time::now();
info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);

View File

@@ -100,16 +100,23 @@ fn test_build_sequence_sets() {
async fn check_target_folder_combination(
folder: &str,
mvbox_move: bool,
chat_msg: bool,
expected_destination: &str,
accepted_chat: bool,
outgoing: bool,
) -> Result<()> {
println!(
"Testing: For folder {folder}, chat_msg {chat_msg}, accepted {accepted_chat}, outgoing {outgoing}"
"Testing: For folder {folder}, mvbox_move {mvbox_move}, chat_msg {chat_msg}, accepted {accepted_chat}, outgoing {outgoing}"
);
let t = TestContext::new_alice().await;
t.ctx
.set_config(Config::ConfiguredMvboxFolder, Some("DeltaChat"))
.await?;
t.ctx
.set_config(Config::MvboxMove, Some(if mvbox_move { "1" } else { "0" }))
.await?;
if accepted_chat {
let contact_id = Contact::create(&t.ctx, "", "bob@example.net").await?;
@@ -154,42 +161,64 @@ async fn check_target_folder_combination(
assert_eq!(
expected,
actual.as_deref(),
"For folder {folder}, chat_msg {chat_msg}, accepted {accepted_chat}, outgoing {outgoing}: expected {expected:?}, got {actual:?}"
"For folder {folder}, mvbox_move {mvbox_move}, chat_msg {chat_msg}, accepted {accepted_chat}, outgoing {outgoing}: expected {expected:?}, got {actual:?}"
);
Ok(())
}
// chat_msg means that the message was sent by Delta Chat
// The tuples are (folder, chat_msg, expected_destination)
const COMBINATIONS_ACCEPTED_CHAT: &[(&str, bool, &str)] = &[
("INBOX", false, "INBOX"),
("INBOX", true, "INBOX"),
("Spam", false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs
("Spam", true, "INBOX"),
// The tuples are (folder, mvbox_move, chat_msg, expected_destination)
const COMBINATIONS_ACCEPTED_CHAT: &[(&str, bool, bool, &str)] = &[
("INBOX", false, false, "INBOX"),
("INBOX", false, true, "INBOX"),
("INBOX", true, false, "INBOX"),
("INBOX", true, true, "DeltaChat"),
("Spam", false, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs
("Spam", false, true, "INBOX"),
("Spam", true, false, "INBOX"), // Move classical emails in accepted chats from Spam to Inbox, not 100% sure on this, we could also just never move non-chat-msgs
("Spam", true, true, "DeltaChat"),
];
// These are the same as above, but non-chat messages in Spam stay in Spam
const COMBINATIONS_REQUEST: &[(&str, bool, &str)] = &[
("INBOX", false, "INBOX"),
("INBOX", true, "INBOX"),
("Spam", false, "Spam"),
("Spam", true, "INBOX"),
const COMBINATIONS_REQUEST: &[(&str, bool, bool, &str)] = &[
("INBOX", false, false, "INBOX"),
("INBOX", false, true, "INBOX"),
("INBOX", true, false, "INBOX"),
("INBOX", true, true, "DeltaChat"),
("Spam", false, false, "Spam"),
("Spam", false, true, "INBOX"),
("Spam", true, false, "Spam"),
("Spam", true, true, "DeltaChat"),
];
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_target_folder_incoming_accepted() -> Result<()> {
for (folder, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
check_target_folder_combination(folder, *chat_msg, expected_destination, true, false)
.await?;
for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
check_target_folder_combination(
folder,
*mvbox_move,
*chat_msg,
expected_destination,
true,
false,
)
.await?;
}
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_target_folder_incoming_request() -> Result<()> {
for (folder, chat_msg, expected_destination) in COMBINATIONS_REQUEST {
check_target_folder_combination(folder, *chat_msg, expected_destination, false, false)
.await?;
for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_REQUEST {
check_target_folder_combination(
folder,
*mvbox_move,
*chat_msg,
expected_destination,
false,
false,
)
.await?;
}
Ok(())
}
@@ -197,9 +226,16 @@ async fn test_target_folder_incoming_request() -> Result<()> {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_target_folder_outgoing() -> Result<()> {
// Test outgoing emails
for (folder, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
check_target_folder_combination(folder, *chat_msg, expected_destination, true, true)
.await?;
for (folder, mvbox_move, chat_msg, expected_destination) in COMBINATIONS_ACCEPTED_CHAT {
check_target_folder_combination(
folder,
*mvbox_move,
*chat_msg,
expected_destination,
true,
true,
)
.await?;
}
Ok(())
}

View File

@@ -56,37 +56,9 @@ pub enum EnteredCertificateChecks {
AcceptInvalidCertificates2 = 3,
}
/// Login parameters for a single IMAP server.
/// Login parameters for a single server, either IMAP or SMTP
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnteredImapLoginParam {
/// Server hostname or IP address.
pub server: String,
/// Server port.
///
/// 0 if not specified.
pub port: u16,
/// Folder to watch.
///
/// If empty, user has not entered anything and it shuold expand to "INBOX" later.
pub folder: String,
/// Socket security.
pub security: Socket,
/// Username.
///
/// Empty string if not specified.
pub user: String,
/// Password.
pub password: String,
}
/// Login parameters for a single SMTP server.
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct EnteredSmtpLoginParam {
pub struct EnteredServerLoginParam {
/// Server hostname or IP address.
pub server: String,
@@ -124,10 +96,10 @@ pub struct EnteredLoginParam {
pub addr: String,
/// IMAP settings.
pub imap: EnteredImapLoginParam,
pub imap: EnteredServerLoginParam,
/// SMTP settings.
pub smtp: EnteredSmtpLoginParam,
pub smtp: EnteredServerLoginParam,
/// TLS options: whether to allow invalid certificates and/or
/// invalid hostnames
@@ -139,8 +111,6 @@ pub struct EnteredLoginParam {
impl EnteredLoginParam {
/// Loads entered account settings.
///
/// This is a legacy API for loading from separate config parameters.
pub(crate) async fn load(context: &Context) -> Result<Self> {
let addr = context
.get_config(Config::Addr)
@@ -157,10 +127,6 @@ impl EnteredLoginParam {
.get_config_parsed::<u16>(Config::MailPort)
.await?
.unwrap_or_default();
// There is no way to set custom folder with this legacy API.
let mail_folder = String::new();
let mail_security = context
.get_config_parsed::<i32>(Config::MailSecurity)
.await?
@@ -219,15 +185,14 @@ impl EnteredLoginParam {
Ok(EnteredLoginParam {
addr,
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
server: mail_server,
port: mail_port,
folder: mail_folder,
security: mail_security,
user: mail_user,
password: mail_pw,
},
smtp: EnteredSmtpLoginParam {
smtp: EnteredServerLoginParam {
server: send_server,
port: send_port,
security: send_security,
@@ -389,15 +354,14 @@ mod tests {
let t = TestContext::new().await;
let param = EnteredLoginParam {
addr: "alice@example.org".to_string(),
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
server: "".to_string(),
port: 0,
folder: "".to_string(),
security: Socket::Starttls,
user: "".to_string(),
password: "foobar".to_string(),
},
smtp: EnteredSmtpLoginParam {
smtp: EnteredServerLoginParam {
server: "".to_string(),
port: 2947,
security: Socket::default(),

View File

@@ -12,7 +12,7 @@ use tokio_io_timeout::TimeoutStream;
use crate::context::Context;
use crate::net::session::SessionStream;
use crate::net::tls::TlsSessionStore;
use crate::net::tls::{SpkiHashStore, TlsSessionStore};
use crate::sql::Sql;
use crate::tools::time;
@@ -130,6 +130,8 @@ pub(crate) async fn connect_tls_inner(
strict_tls: bool,
alpn: &str,
tls_session_store: &TlsSessionStore,
spki_hash_store: &SpkiHashStore,
sql: &Sql,
) -> Result<impl SessionStream + 'static> {
let use_sni = true;
let tcp_stream = connect_tcp_inner(addr).await?;
@@ -141,6 +143,8 @@ pub(crate) async fn connect_tls_inner(
alpn,
tcp_stream,
tls_session_store,
spki_hash_store,
sql,
)
.await?;
Ok(tls_stream)

View File

@@ -87,6 +87,8 @@ where
"",
proxy_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
Box::new(tls_stream)
@@ -99,6 +101,8 @@ where
"",
tcp_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
Box::new(tls_stream)

View File

@@ -439,6 +439,8 @@ impl ProxyConfig {
"",
tcp_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
let auth = if let Some((username, password)) = &https_config.user_password {

View File

@@ -6,13 +6,19 @@ use std::sync::Arc;
use anyhow::Result;
use crate::net::session::SessionStream;
use crate::sql::Sql;
use tokio_rustls::rustls;
use tokio_rustls::rustls::client::ClientSessionStore;
use tokio_rustls::rustls::server::ParsedCertificate;
mod danger;
use danger::NoCertificateVerification;
use danger::CustomCertificateVerifier;
mod spki;
pub use spki::SpkiHashStore;
#[expect(clippy::too_many_arguments)]
pub async fn wrap_tls<'a>(
strict_tls: bool,
hostname: &str,
@@ -21,10 +27,21 @@ pub async fn wrap_tls<'a>(
alpn: &str,
stream: impl SessionStream + 'static,
tls_session_store: &TlsSessionStore,
spki_hash_store: &SpkiHashStore,
sql: &Sql,
) -> Result<impl SessionStream + 'a> {
if strict_tls {
let tls_stream =
wrap_rustls(hostname, port, use_sni, alpn, stream, tls_session_store).await?;
let tls_stream = wrap_rustls(
hostname,
port,
use_sni,
alpn,
stream,
tls_session_store,
spki_hash_store,
sql,
)
.await?;
let boxed_stream: Box<dyn SessionStream> = Box::new(tls_stream);
Ok(boxed_stream)
} else {
@@ -94,6 +111,7 @@ impl TlsSessionStore {
}
}
#[expect(clippy::too_many_arguments)]
pub async fn wrap_rustls<'a>(
hostname: &str,
port: u16,
@@ -101,6 +119,8 @@ pub async fn wrap_rustls<'a>(
alpn: &str,
stream: impl SessionStream + 'a,
tls_session_store: &TlsSessionStore,
spki_hash_store: &SpkiHashStore,
sql: &Sql,
) -> Result<impl SessionStream + 'a> {
let mut root_cert_store = rustls::RootCertStore::empty();
root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
@@ -127,20 +147,27 @@ pub async fn wrap_rustls<'a>(
config.resumption = resumption;
config.enable_sni = use_sni;
// Do not verify certificates for hostnames starting with `_`.
// They are used for servers with self-signed certificates, e.g. for local testing.
// Hostnames starting with `_` can have only self-signed TLS certificates or wildcard certificates.
// It is not possible to get valid non-wildcard TLS certificates because CA/Browser Forum requirements
// explicitly state that domains should start with a letter, digit or hyphen:
// https://github.com/cabforum/servercert/blob/24f38fd4765e019db8bb1a8c56bf63c7115ce0b0/docs/BR.md
if hostname.starts_with("_") {
config
.dangerous()
.set_certificate_verifier(Arc::new(NoCertificateVerification::new()));
}
config
.dangerous()
.set_certificate_verifier(Arc::new(CustomCertificateVerifier::new(
spki_hash_store.get_spki_hash(hostname, sql).await?,
)));
let tls = tokio_rustls::TlsConnector::from(Arc::new(config));
let name = tokio_rustls::rustls::pki_types::ServerName::try_from(hostname)?.to_owned();
let tls_stream = tls.connect(name, stream).await?;
// Successfully connected.
// Remember SPKI hash to accept it later if certificate expires.
let (_io, client_connection) = tls_stream.get_ref();
if let Some(end_entity) = client_connection
.peer_certificates()
.and_then(|certs| certs.first())
{
let parsed_certificate = ParsedCertificate::try_from(end_entity)?;
let spki = parsed_certificate.subject_public_key_info();
spki_hash_store.save_spki(hostname, &spki, sql).await?;
}
Ok(tls_stream)
}

View File

@@ -1,26 +1,85 @@
//! Dangerous TLS implementation of accepting invalid certificates for Rustls.
//! Custom TLS verification.
//!
//! We want to accept expired certificates.
use rustls::RootCertStore;
use rustls::client::{verify_server_cert_signed_by_trust_anchor, verify_server_name};
use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
use rustls::server::ParsedCertificate;
use tokio_rustls::rustls;
#[derive(Debug)]
pub(super) struct NoCertificateVerification();
use crate::net::tls::spki::spki_hash;
impl NoCertificateVerification {
pub(super) fn new() -> Self {
Self()
#[derive(Debug)]
pub(super) struct CustomCertificateVerifier {
/// Root certificates.
root_cert_store: RootCertStore,
/// Expected SPKI hash as a base64 of SHA-256.
spki_hash: Option<String>,
}
impl CustomCertificateVerifier {
pub(super) fn new(spki_hash: Option<String>) -> Self {
let mut root_cert_store = rustls::RootCertStore::empty();
root_cert_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
Self {
root_cert_store,
spki_hash,
}
}
}
impl rustls::client::danger::ServerCertVerifier for NoCertificateVerification {
impl rustls::client::danger::ServerCertVerifier for CustomCertificateVerifier {
fn verify_server_cert(
&self,
_end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &ServerName<'_>,
end_entity: &CertificateDer<'_>,
intermediates: &[CertificateDer<'_>],
server_name: &ServerName<'_>,
_ocsp_response: &[u8],
_now: UnixTime,
now: UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
let parsed_certificate = ParsedCertificate::try_from(end_entity)?;
let spki = parsed_certificate.subject_public_key_info();
let provider = rustls::crypto::ring::default_provider();
if let ServerName::DnsName(dns_name) = server_name
&& dns_name.as_ref().starts_with("_")
{
// Do not verify certificates for hostnames starting with `_`.
// They are used for servers with self-signed certificates, e.g. for local testing.
// Hostnames starting with `_` can have only self-signed TLS certificates or wildcard certificates.
// It is not possible to get valid non-wildcard TLS certificates because CA/Browser Forum requirements
// explicitly state that domains should start with a letter, digit or hyphen:
// https://github.com/cabforum/servercert/blob/24f38fd4765e019db8bb1a8c56bf63c7115ce0b0/docs/BR.md
} else if let Some(hash) = &self.spki_hash
&& spki_hash(&spki) == *hash
{
// Last time we successfully connected to this hostname with TLS checks,
// SPKI had this hash.
// It does not matter if certificate has now expired.
} else {
// verify_server_cert_signed_by_trust_anchor does no revocation checking:
// <https://docs.rs/rustls/0.23.37/rustls/client/fn.verify_server_cert_signed_by_trust_anchor.html>
// We don't do it either.
verify_server_cert_signed_by_trust_anchor(
&parsed_certificate,
&self.root_cert_store,
intermediates,
now,
provider.signature_verification_algorithms.all,
)?;
}
// Verify server name unconditionally.
//
// We do this even for self-signed certificates when hostname starts with `_`
// so we don't try to connect to captive portals
// and fail on MITM certificates if they are generated once
// and reused for all hostnames.
verify_server_name(&parsed_certificate, server_name)?;
Ok(rustls::client::danger::ServerCertVerified::assertion())
}

92
src/net/tls/spki.rs Normal file
View File

@@ -0,0 +1,92 @@
//! SPKI hash storage.
//!
//! We store hashes of Subject Public Key Info from TLS certificates
//! after successful connection to allow connecting when
//! server certificate expires as long as the key is not changed.
use std::collections::BTreeMap;
use anyhow::Result;
use base64::Engine as _;
use parking_lot::RwLock;
use sha2::{Digest, Sha256};
use tokio_rustls::rustls::pki_types::SubjectPublicKeyInfoDer;
use crate::sql::Sql;
/// Calculates Subject Public Key Info SHA-256 hash and returns it as base64.
///
/// This is the same format as used in <https://www.rfc-editor.org/rfc/rfc7469>.
/// You can calculate the same hash for any remote host with
/// ```sh
/// openssl s_client -connect "$HOST:993" -servername "$HOST" </dev/null 2>/dev/null |
/// openssl x509 -pubkey -noout |
/// openssl pkey -pubin -outform der |
/// openssl dgst -sha256 -binary |
/// openssl enc -base64
/// ```
pub fn spki_hash(spki: &SubjectPublicKeyInfoDer) -> String {
let spki_hash = Sha256::digest(spki);
base64::engine::general_purpose::STANDARD.encode(spki_hash)
}
/// Write-through cache for SPKI hashes.
#[derive(Debug)]
pub struct SpkiHashStore {
/// Map from hostnames to base64 of SHA-256 hashes.
pub hash_store: RwLock<BTreeMap<String, String>>,
}
impl SpkiHashStore {
pub fn new() -> Self {
Self {
hash_store: RwLock::new(BTreeMap::new()),
}
}
/// Returns base64 of SPKI hash if we have previously successfully connected to given hostname.
pub async fn get_spki_hash(&self, hostname: &str, sql: &Sql) -> Result<Option<String>> {
if let Some(hash) = self.hash_store.read().get(hostname).cloned() {
return Ok(Some(hash));
}
match sql
.query_row_optional(
"SELECT spki_hash FROM tls_spki WHERE host=?",
(hostname,),
|row| {
let spki_hash: String = row.get(0)?;
Ok(spki_hash)
},
)
.await?
{
Some(hash) => {
self.hash_store
.write()
.insert(hostname.to_string(), hash.clone());
Ok(Some(hash))
}
None => Ok(None),
}
}
/// Saves SPKI hash after successful connection.
pub async fn save_spki(
&self,
hostname: &str,
spki: &SubjectPublicKeyInfoDer<'_>,
sql: &Sql,
) -> Result<()> {
let hash = spki_hash(spki);
self.hash_store
.write()
.insert(hostname.to_string(), hash.clone());
sql.execute(
"INSERT OR REPLACE INTO tls_spki (host, spki_hash) VALUES (?, ?)",
(hostname, hash),
)
.await?;
Ok(())
}
}

View File

@@ -17,7 +17,7 @@ use crate::config::Config;
use crate::contact::{Contact, ContactId, Origin};
use crate::context::Context;
use crate::key::Fingerprint;
use crate::login_param::{EnteredCertificateChecks, EnteredImapLoginParam, EnteredLoginParam};
use crate::login_param::{EnteredCertificateChecks, EnteredLoginParam, EnteredServerLoginParam};
use crate::net::http::post_empty;
use crate::net::proxy::{DEFAULT_SOCKS_PORT, ProxyConfig};
use crate::token;
@@ -41,7 +41,7 @@ pub(crate) const DCBACKUP_SCHEME_PREFIX: &str = "DCBACKUP";
/// Version written to Backups and Backup-QR-Codes.
/// Imports will fail when they have a larger version.
pub(crate) const DCBACKUP_VERSION: i32 = 5;
pub(crate) const DCBACKUP_VERSION: i32 = 4;
/// Scanned QR code.
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -824,7 +824,7 @@ pub(crate) async fn login_param_from_account_qr(
let param = EnteredLoginParam {
addr,
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
password,
..Default::default()
},
@@ -844,7 +844,7 @@ pub(crate) async fn login_param_from_account_qr(
let param = EnteredLoginParam {
addr: email,
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
password,
..Default::default()
},

View File

@@ -5,9 +5,7 @@ use anyhow::{Context as _, Result, bail};
use deltachat_contact_tools::may_be_valid_addr;
use super::{DCLOGIN_SCHEME, Qr};
use crate::login_param::{
EnteredCertificateChecks, EnteredImapLoginParam, EnteredLoginParam, EnteredSmtpLoginParam,
};
use crate::login_param::{EnteredCertificateChecks, EnteredLoginParam, EnteredServerLoginParam};
use crate::provider::Socket;
/// Options for `dclogin:` scheme.
@@ -180,15 +178,14 @@ pub(crate) fn login_param_from_login_qr(
} => {
let param = EnteredLoginParam {
addr: addr.to_string(),
imap: EnteredImapLoginParam {
imap: EnteredServerLoginParam {
server: imap_host.unwrap_or_default(),
port: imap_port.unwrap_or_default(),
folder: "INBOX".to_string(),
security: imap_security.unwrap_or_default(),
user: imap_username.unwrap_or_default(),
password: imap_password.unwrap_or(mail_pw),
},
smtp: EnteredSmtpLoginParam {
smtp: EnteredServerLoginParam {
server: smtp_host.unwrap_or_default(),
port: smtp_port.unwrap_or_default(),
security: smtp_security.unwrap_or_default(),

View File

@@ -9,6 +9,7 @@ use async_imap::types::{Quota, QuotaResource};
use crate::chat::add_device_msg_with_importance;
use crate::config::Config;
use crate::context::Context;
use crate::imap::get_watched_folders;
use crate::imap::session::Session as ImapSession;
use crate::log::warn;
use crate::message::Message;
@@ -47,24 +48,26 @@ pub struct QuotaInfo {
async fn get_unique_quota_roots_and_usage(
session: &mut ImapSession,
folder: &str,
folders: Vec<String>,
) -> Result<BTreeMap<String, Vec<QuotaResource>>> {
let mut unique_quota_roots: BTreeMap<String, Vec<QuotaResource>> = BTreeMap::new();
let (quota_roots, quotas) = &session.get_quota_root(folder).await?;
// if there are new quota roots found in this imap folder, add them to the list
for qr_entries in quota_roots {
for quota_root_name in &qr_entries.quota_root_names {
// the quota for that quota root
let quota: Quota = quotas
.iter()
.find(|q| &q.root_name == quota_root_name)
.cloned()
.context("quota_root should have a quota")?;
// replace old quotas, because between fetching quotaroots for folders,
// messages could be received and so the usage could have been changed
*unique_quota_roots
.entry(quota_root_name.clone())
.or_default() = quota.resources;
for folder in folders {
let (quota_roots, quotas) = &session.get_quota_root(&folder).await?;
// if there are new quota roots found in this imap folder, add them to the list
for qr_entries in quota_roots {
for quota_root_name in &qr_entries.quota_root_names {
// the quota for that quota root
let quota: Quota = quotas
.iter()
.find(|q| &q.root_name == quota_root_name)
.cloned()
.context("quota_root should have a quota")?;
// replace old quotas, because between fetching quotaroots for folders,
// messages could be received and so the usage could have been changed
*unique_quota_roots
.entry(quota_root_name.clone())
.or_default() = quota.resources;
}
}
}
Ok(unique_quota_roots)
@@ -119,17 +122,14 @@ impl Context {
/// As the message is added only once, the user is not spammed
/// in case for some providers the quota is always at ~100%
/// and new space is allocated as needed.
pub(crate) async fn update_recent_quota(
&self,
session: &mut ImapSession,
folder: &str,
) -> Result<()> {
pub(crate) async fn update_recent_quota(&self, session: &mut ImapSession) -> Result<()> {
let transport_id = session.transport_id();
info!(self, "Transport {transport_id}: Updating quota.");
let quota = if session.can_check_quota() {
get_unique_quota_roots_and_usage(session, folder).await
let folders = get_watched_folders(self).await?;
get_unique_quota_roots_and_usage(session, folders).await
} else {
Err(anyhow!(stock_str::not_supported_by_provider(self)))
};

View File

@@ -17,7 +17,7 @@ use crate::context::Context;
use crate::download::{download_known_post_messages_without_pre_message, download_msgs};
use crate::ephemeral::{self, delete_expired_imap_messages};
use crate::events::EventType;
use crate::imap::{Imap, session::Session};
use crate::imap::{FolderMeaning, Imap, session::Session};
use crate::location;
use crate::log::{LogExt, warn};
use crate::smtp::{Smtp, send_smtp_messages};
@@ -211,19 +211,25 @@ impl SchedulerState {
/// Indicate that the network likely has come back.
pub(crate) async fn maybe_network(&self) {
let inner = self.inner.read().await;
let inboxes = match *inner {
let (inboxes, oboxes) = match *inner {
InnerSchedulerState::Started(ref scheduler) => {
scheduler.maybe_network();
scheduler
let inboxes = scheduler
.inboxes
.iter()
.map(|b| b.conn_state.state.connectivity.clone())
.collect::<Vec<_>>()
.collect::<Vec<_>>();
let oboxes = scheduler
.oboxes
.iter()
.map(|b| b.conn_state.state.connectivity.clone())
.collect::<Vec<_>>();
(inboxes, oboxes)
}
_ => return,
};
drop(inner);
connectivity::idle_interrupted(inboxes);
connectivity::idle_interrupted(inboxes, oboxes);
}
/// Indicate that the network likely is lost.
@@ -312,10 +318,7 @@ impl Drop for IoPausedGuard {
struct SchedBox {
/// Address at the used chatmail/email relay
addr: String,
/// Folder name
folder: String,
meaning: FolderMeaning,
conn_state: ImapConnectionState,
/// IMAP loop task handle.
@@ -327,6 +330,8 @@ struct SchedBox {
pub(crate) struct Scheduler {
/// Inboxes, one per transport.
inboxes: Vec<SchedBox>,
/// Optional boxes -- mvbox.
oboxes: Vec<SchedBox>,
smtp: SmtpConnectionState,
smtp_handle: task::JoinHandle<()>,
ephemeral_handle: task::JoinHandle<()>,
@@ -407,12 +412,42 @@ async fn inbox_loop(
.await;
}
/// Convert folder meaning
/// used internally by [fetch_idle] and [Context::background_fetch].
///
/// Returns folder configuration key and folder name
/// if such folder is configured, `Ok(None)` otherwise.
pub async fn convert_folder_meaning(
ctx: &Context,
folder_meaning: FolderMeaning,
) -> Result<Option<(Config, String)>> {
let folder_config = match folder_meaning.to_config() {
Some(c) => c,
None => {
// Such folder cannot be configured,
// e.g. a `FolderMeaning::Spam` folder.
return Ok(None);
}
};
let folder = ctx
.get_config(folder_config)
.await
.with_context(|| format!("Failed to retrieve {folder_config} folder"))?;
if let Some(watch_folder) = folder {
Ok(Some((folder_config, watch_folder)))
} else {
Ok(None)
}
}
async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) -> Result<Session> {
let transport_id = session.transport_id();
// Update quota no more than once a minute.
if ctx.quota_needs_update(session.transport_id(), 60).await
&& let Err(err) = ctx.update_recent_quota(&mut session, &imap.folder).await
&& let Err(err) = ctx.update_recent_quota(&mut session).await
{
warn!(
ctx,
@@ -459,7 +494,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session)
.await
.context("Failed to register push token")?;
let session = fetch_idle(ctx, imap, session).await?;
let session = fetch_idle(ctx, imap, session, FolderMeaning::Inbox).await?;
Ok(session)
}
@@ -468,19 +503,34 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session)
/// This function performs all IMAP operations on a single folder, selecting it if necessary and
/// handling all the errors. In case of an error, an error is returned and connection is dropped,
/// otherwise connection is returned.
async fn fetch_idle(ctx: &Context, connection: &mut Imap, mut session: Session) -> Result<Session> {
async fn fetch_idle(
ctx: &Context,
connection: &mut Imap,
mut session: Session,
folder_meaning: FolderMeaning,
) -> Result<Session> {
let transport_id = session.transport_id();
let watch_folder = connection.folder.clone();
let Some((folder_config, watch_folder)) = convert_folder_meaning(ctx, folder_meaning).await?
else {
// The folder is not configured.
// For example, this happens if the server does not have Sent folder
// but watching Sent folder is enabled.
connection.connectivity.set_not_configured(ctx);
connection.idle_interrupt_receiver.recv().await.ok();
bail!("Cannot fetch folder {folder_meaning} because it is not configured");
};
session
.store_seen_flags_on_imap(ctx)
.await
.context("store_seen_flags_on_imap")?;
if folder_config == Config::ConfiguredInboxFolder {
session
.store_seen_flags_on_imap(ctx)
.await
.context("store_seen_flags_on_imap")?;
}
// Fetch the watched folder.
connection
.fetch_move_delete(ctx, &mut session, &watch_folder)
.fetch_move_delete(ctx, &mut session, &watch_folder, folder_meaning)
.await
.context("fetch_move_delete")?;
@@ -514,7 +564,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, mut session: Session)
ctx,
"Transport {transport_id}: IMAP session does not support IDLE, going to fake idle."
);
connection.fake_idle(ctx, &watch_folder).await?;
connection.fake_idle(ctx, watch_folder).await?;
return Ok(session);
}
@@ -529,7 +579,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, mut session: Session)
ctx,
"Transport {transport_id}: IMAP IDLE is disabled, going to fake idle."
);
connection.fake_idle(ctx, &watch_folder).await?;
connection.fake_idle(ctx, watch_folder).await?;
return Ok(session);
}
@@ -545,6 +595,69 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, mut session: Session)
Ok(session)
}
/// Simplified IMAP loop to watch non-inbox folders.
async fn simple_imap_loop(
ctx: Context,
started: oneshot::Sender<()>,
inbox_handlers: ImapConnectionHandlers,
folder_meaning: FolderMeaning,
) {
use futures::future::FutureExt;
info!(ctx, "Starting simple loop for {folder_meaning}.");
let ImapConnectionHandlers {
mut connection,
stop_token,
} = inbox_handlers;
let ctx1 = ctx.clone();
let fut = async move {
let ctx = ctx1;
if let Err(()) = started.send(()) {
warn!(
ctx,
"Simple imap loop for {folder_meaning}, missing started receiver."
);
return;
}
let mut old_session: Option<Session> = None;
loop {
let session = if let Some(session) = old_session.take() {
session
} else {
info!(ctx, "Preparing new IMAP session for {folder_meaning}.");
match connection.prepare(&ctx).await {
Err(err) => {
warn!(
ctx,
"Failed to prepare {folder_meaning} connection: {err:#}."
);
continue;
}
Ok(session) => session,
}
};
match fetch_idle(&ctx, &mut connection, session, folder_meaning).await {
Err(err) => warn!(ctx, "Failed fetch_idle: {err:#}"),
Ok(session) => {
old_session = Some(session);
}
}
}
};
stop_token
.cancelled()
.map(|_| {
info!(ctx, "Shutting down IMAP loop for {folder_meaning}.");
})
.race(fut)
.await;
}
async fn smtp_loop(
ctx: Context,
started: oneshot::Sender<()>,
@@ -647,6 +760,7 @@ impl Scheduler {
let (location_interrupt_send, location_interrupt_recv) = channel::bounded(1);
let mut inboxes = Vec::new();
let mut oboxes = Vec::new();
let mut start_recvs = Vec::new();
for (transport_id, configured_login_param) in ConfiguredLoginParam::load_all(ctx).await? {
@@ -658,17 +772,30 @@ impl Scheduler {
task::spawn(inbox_loop(ctx, inbox_start_send, inbox_handlers))
};
let addr = configured_login_param.addr.clone();
let folder = configured_login_param
.imap_folder
.unwrap_or_else(|| "INBOX".to_string());
let inbox = SchedBox {
addr: addr.clone(),
folder,
meaning: FolderMeaning::Inbox,
conn_state,
handle,
};
inboxes.push(inbox);
start_recvs.push(inbox_start_recv);
if ctx.should_watch_mvbox().await? {
let (conn_state, handlers) =
ImapConnectionState::new(ctx, transport_id, configured_login_param).await?;
let (start_send, start_recv) = oneshot::channel();
let ctx = ctx.clone();
let meaning = FolderMeaning::Mvbox;
let handle = task::spawn(simple_imap_loop(ctx, start_send, handlers, meaning));
oboxes.push(SchedBox {
addr,
meaning,
conn_state,
handle,
});
start_recvs.push(start_recv);
}
}
let smtp_handle = {
@@ -695,6 +822,7 @@ impl Scheduler {
let res = Self {
inboxes,
oboxes,
smtp,
smtp_handle,
ephemeral_handle,
@@ -714,7 +842,7 @@ impl Scheduler {
}
fn boxes(&self) -> impl Iterator<Item = &SchedBox> {
self.inboxes.iter()
self.inboxes.iter().chain(self.oboxes.iter())
}
fn maybe_network(&self) {
@@ -768,7 +896,7 @@ impl Scheduler {
let timeout_duration = std::time::Duration::from_secs(30);
let tracker = TaskTracker::new();
for b in self.inboxes {
for b in self.inboxes.into_iter().chain(self.oboxes) {
let context = context.clone();
tracker.spawn(async move {
tokio::time::timeout(timeout_duration, b.handle)

View File

@@ -5,10 +5,11 @@ use std::{iter::once, ops::Deref, sync::Arc};
use anyhow::Result;
use humansize::{BINARY, format_size};
use crate::context::Context;
use crate::events::EventType;
use crate::imap::{FolderMeaning, get_watched_folder_configs};
use crate::quota::{QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_WARN_THRESHOLD_PERCENTAGE};
use crate::stock_str;
use crate::{context::Context, log::LogExt};
use super::InnerSchedulerState;
@@ -66,33 +67,40 @@ enum DetailedConnectivity {
/// Connection is established and is idle.
Idle,
/// The folder was configured not to be watched or configured_*_folder is not set
NotConfigured,
}
impl DetailedConnectivity {
fn to_basic(&self) -> Connectivity {
fn to_basic(&self) -> Option<Connectivity> {
match self {
DetailedConnectivity::Error(_) => Connectivity::NotConnected,
DetailedConnectivity::Uninitialized => Connectivity::NotConnected,
DetailedConnectivity::Connecting => Connectivity::Connecting,
DetailedConnectivity::Working => Connectivity::Working,
DetailedConnectivity::InterruptingIdle => Connectivity::Working,
DetailedConnectivity::Error(_) => Some(Connectivity::NotConnected),
DetailedConnectivity::Uninitialized => Some(Connectivity::NotConnected),
DetailedConnectivity::Connecting => Some(Connectivity::Connecting),
DetailedConnectivity::Working => Some(Connectivity::Working),
DetailedConnectivity::InterruptingIdle => Some(Connectivity::Working),
// At this point IMAP has just connected,
// but does not know yet if there are messages to download.
// We still convert this to Working state
// so user can see "Updating..." and not "Connected"
// which is reserved for idle state.
DetailedConnectivity::Preparing => Connectivity::Working,
DetailedConnectivity::Preparing => Some(Connectivity::Working),
DetailedConnectivity::Idle => Connectivity::Connected,
// Just don't return a connectivity, probably the folder is configured not to be
// watched, so we are not interested in it.
DetailedConnectivity::NotConfigured => None,
DetailedConnectivity::Idle => Some(Connectivity::Connected),
}
}
fn to_icon(&self) -> String {
match self {
DetailedConnectivity::Error(_) | DetailedConnectivity::Uninitialized => {
"<span class=\"red dot\"></span>".to_string()
}
DetailedConnectivity::Error(_)
| DetailedConnectivity::Uninitialized
| DetailedConnectivity::NotConfigured => "<span class=\"red dot\"></span>".to_string(),
DetailedConnectivity::Connecting => "<span class=\"yellow dot\"></span>".to_string(),
DetailedConnectivity::Preparing
| DetailedConnectivity::Working
@@ -112,6 +120,7 @@ impl DetailedConnectivity {
DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Idle => {
stock_str::connected(context)
}
DetailedConnectivity::NotConfigured => "Not configured".to_string(),
}
}
@@ -130,6 +139,7 @@ impl DetailedConnectivity {
DetailedConnectivity::InterruptingIdle
| DetailedConnectivity::Preparing
| DetailedConnectivity::Idle => stock_str::last_msg_sent_successfully(context),
DetailedConnectivity::NotConfigured => "Not configured".to_string(),
}
}
@@ -141,6 +151,7 @@ impl DetailedConnectivity {
DetailedConnectivity::Working => false,
DetailedConnectivity::InterruptingIdle => false,
DetailedConnectivity::Preparing => false, // Just connected, there may still be work to do.
DetailedConnectivity::NotConfigured => true,
DetailedConnectivity::Idle => true,
}
}
@@ -169,6 +180,9 @@ impl ConnectivityStore {
pub(crate) fn set_preparing(&self, context: &Context) {
self.set(context, DetailedConnectivity::Preparing);
}
pub(crate) fn set_not_configured(&self, context: &Context) {
self.set(context, DetailedConnectivity::NotConfigured);
}
pub(crate) fn set_idle(&self, context: &Context) {
self.set(context, DetailedConnectivity::Idle);
}
@@ -176,7 +190,7 @@ impl ConnectivityStore {
fn get_detailed(&self) -> DetailedConnectivity {
self.0.lock().deref().clone()
}
fn get_basic(&self) -> Connectivity {
fn get_basic(&self) -> Option<Connectivity> {
self.0.lock().to_basic()
}
fn get_all_work_done(&self) -> bool {
@@ -187,14 +201,27 @@ impl ConnectivityStore {
/// Set all folder states to InterruptingIdle in case they were `Idle` before.
/// Called during `dc_maybe_network()` to make sure that `all_work_done()`
/// returns false immediately after `dc_maybe_network()`.
pub(crate) fn idle_interrupted(inboxes: Vec<ConnectivityStore>) {
pub(crate) fn idle_interrupted(inboxes: Vec<ConnectivityStore>, oboxes: Vec<ConnectivityStore>) {
for inbox in inboxes {
let mut connectivity_lock = inbox.0.lock();
if *connectivity_lock == DetailedConnectivity::Idle {
// For the inbox, we also have to set the connectivity to InterruptingIdle if it was
// NotConfigured before: If all folders are NotConfigured, dc_get_connectivity()
// returns Connected. But after dc_maybe_network(), dc_get_connectivity() must not
// return Connected until DC is completely done with fetching folders; this also
// includes scan_folders() which happens on the inbox thread.
if *connectivity_lock == DetailedConnectivity::Idle
|| *connectivity_lock == DetailedConnectivity::NotConfigured
{
*connectivity_lock = DetailedConnectivity::InterruptingIdle;
}
}
for state in oboxes {
let mut connectivity_lock = state.0.lock();
if *connectivity_lock == DetailedConnectivity::Idle {
*connectivity_lock = DetailedConnectivity::InterruptingIdle;
}
}
// No need to send ConnectivityChanged, the user-facing connectivity doesn't change because
// of what we do here.
}
@@ -207,7 +234,9 @@ pub(crate) fn maybe_network_lost(context: &Context, stores: Vec<ConnectivityStor
let mut connectivity_lock = store.0.lock();
if !matches!(
*connectivity_lock,
DetailedConnectivity::Uninitialized | DetailedConnectivity::Error(_)
DetailedConnectivity::Uninitialized
| DetailedConnectivity::Error(_)
| DetailedConnectivity::NotConfigured,
) {
*connectivity_lock = DetailedConnectivity::Error("Connection lost".to_string());
}
@@ -244,8 +273,9 @@ impl Context {
let stores = self.connectivities.lock().clone();
let mut connectivities = Vec::new();
for s in stores {
let connectivity = s.get_basic();
connectivities.push(connectivity);
if let Some(connectivity) = s.get_basic() {
connectivities.push(connectivity);
}
}
connectivities
.into_iter()
@@ -356,7 +386,7 @@ impl Context {
.map(|b| {
(
b.addr.clone(),
b.folder.clone(),
b.meaning,
b.conn_state.state.connectivity.clone(),
)
})
@@ -381,6 +411,7 @@ impl Context {
// [======67%===== ]
// =============================================================================================
let watched_folders = get_watched_folder_configs(self).await?;
let incoming_messages = stock_str::incoming_messages(self);
ret += &format!("<h3>{incoming_messages}</h3><ul>");
@@ -402,14 +433,41 @@ impl Context {
let folders = folders_states
.iter()
.filter(|(folder_addr, ..)| *folder_addr == transport_addr);
for (_addr, _folder, state) in folders {
let detailed = &state.get_detailed();
ret += &*detailed.to_icon();
ret += " <b>";
ret += &*domain_escaped;
ret += ":</b> ";
ret += &*escaper::encode_minimal(&detailed.to_string_imap(self));
ret += "<br />";
for (_addr, folder, state) in folders {
let mut folder_added = false;
if let Some(config) = folder.to_config().filter(|c| watched_folders.contains(c)) {
let f = self.get_config(config).await.log_err(self).ok().flatten();
if let Some(foldername) = f {
let detailed = &state.get_detailed();
ret += &*detailed.to_icon();
ret += " <b>";
if folder == &FolderMeaning::Inbox {
ret += &*domain_escaped;
} else {
ret += &*escaper::encode_minimal(&foldername);
}
ret += ":</b> ";
ret += &*escaper::encode_minimal(&detailed.to_string_imap(self));
ret += "<br />";
folder_added = true;
}
}
if !folder_added && folder == &FolderMeaning::Inbox {
let detailed = &state.get_detailed();
if let DetailedConnectivity::Error(_) = detailed {
// On the inbox thread, we also do some other things like scan_folders and run jobs
// so, maybe, the inbox is not watched, but something else went wrong
ret += &*detailed.to_icon();
ret += " ";
ret += &*escaper::encode_minimal(&detailed.to_string_imap(self));
ret += "<br />";
}
}
}
let Some(quota) = quota.get(&transport_id) else {

View File

@@ -11,11 +11,12 @@ use crate::log::warn;
use crate::net::dns::{lookup_host_with_cache, update_connect_timestamp};
use crate::net::proxy::ProxyConfig;
use crate::net::session::SessionBufStream;
use crate::net::tls::{TlsSessionStore, wrap_tls};
use crate::net::tls::{SpkiHashStore, TlsSessionStore, wrap_tls};
use crate::net::{
connect_tcp_inner, connect_tls_inner, run_connection_attempts, update_connection_history,
};
use crate::oauth2::get_oauth2_access_token;
use crate::sql::Sql;
use crate::tools::time;
use crate::transport::ConnectionCandidate;
use crate::transport::ConnectionSecurity;
@@ -111,10 +112,26 @@ async fn connection_attempt(
);
let res = match security {
ConnectionSecurity::Tls => {
connect_secure(resolved_addr, host, strict_tls, &context.tls_session_store).await
connect_secure(
resolved_addr,
host,
strict_tls,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await
}
ConnectionSecurity::Starttls => {
connect_starttls(resolved_addr, host, strict_tls, &context.tls_session_store).await
connect_starttls(
resolved_addr,
host,
strict_tls,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await
}
ConnectionSecurity::Plain => connect_insecure(resolved_addr).await,
};
@@ -240,6 +257,8 @@ async fn connect_secure_proxy(
alpn(port),
proxy_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await?;
let mut buffered_stream = BufStream::new(tls_stream);
@@ -273,6 +292,8 @@ async fn connect_starttls_proxy(
"",
tcp_stream,
&context.tls_session_store,
&context.spki_hash_store,
&context.sql,
)
.await
.context("STARTTLS upgrade failed")?;
@@ -299,6 +320,8 @@ async fn connect_secure(
hostname: &str,
strict_tls: bool,
tls_session_store: &TlsSessionStore,
spki_hash_store: &SpkiHashStore,
sql: &Sql,
) -> Result<Box<dyn SessionBufStream>> {
let tls_stream = connect_tls_inner(
addr,
@@ -306,6 +329,8 @@ async fn connect_secure(
strict_tls,
alpn(addr.port()),
tls_session_store,
spki_hash_store,
sql,
)
.await?;
let mut buffered_stream = BufStream::new(tls_stream);
@@ -319,6 +344,8 @@ async fn connect_starttls(
host: &str,
strict_tls: bool,
tls_session_store: &TlsSessionStore,
spki_hash_store: &SpkiHashStore,
sql: &Sql,
) -> Result<Box<dyn SessionBufStream>> {
let use_sni = false;
let tcp_stream = connect_tcp_inner(addr).await?;
@@ -336,6 +363,8 @@ async fn connect_starttls(
"",
tcp_stream,
tls_session_store,
spki_hash_store,
sql,
)
.await
.context("STARTTLS upgrade failed")?;

View File

@@ -9,6 +9,7 @@ use rusqlite::{Connection, OpenFlags, Row, config::DbConfig, types::ValueRef};
use tokio::sync::RwLock;
use crate::blob::BlobObject;
use crate::chat::add_device_msg;
use crate::config::Config;
use crate::constants::DC_CHAT_ID_TRASH;
use crate::context::Context;
@@ -17,11 +18,13 @@ use crate::ephemeral::start_ephemeral_timers;
use crate::imex::BLOBS_BACKUP_NAME;
use crate::location::delete_orphaned_poi_locations;
use crate::log::{LogExt, warn};
use crate::message::Message;
use crate::message::MsgId;
use crate::net::dns::prune_dns_cache;
use crate::net::http::http_cache_cleanup;
use crate::net::prune_connection_history;
use crate::param::{Param, Params};
use crate::stock_str;
use crate::tools::{SystemTime, delete_file, time};
/// Extension to [`rusqlite::ToSql`] trait
@@ -827,6 +830,12 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
);
}
maybe_add_mvbox_move_deprecation_message(context)
.await
.context("maybe_add_mvbox_move_deprecation_message")
.log_err(context)
.ok();
if let Err(err) = incremental_vacuum(context).await {
warn!(context, "Failed to run incremental vacuum: {err:#}.");
}
@@ -904,6 +913,18 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
Ok(())
}
/// Adds device message about `mvbox_move` config deprecation
/// if the user has it enabled.
async fn maybe_add_mvbox_move_deprecation_message(context: &Context) -> Result<()> {
if !context.get_config_bool(Config::OnlyFetchMvbox).await?
&& context.get_config_bool(Config::MvboxMove).await?
{
let mut msg = Message::new_text(stock_str::mvbox_move_deprecation(context));
add_device_msg(context, Some("mvbox_move_deprecation"), Some(&mut msg)).await?;
}
Ok(())
}
/// Get the value of a column `idx` of the `row` as `Vec<u8>`.
pub fn row_get_vec(row: &Row, idx: usize) -> rusqlite::Result<Vec<u8>> {
row.get(idx).or_else(|err| match row.get_ref(idx)? {

View File

@@ -2318,43 +2318,11 @@ ALTER TABLE contacts ADD COLUMN name_normalized TEXT;
inc_and_check(&mut migration_version, 150)?;
if dbversion < migration_version {
sql.execute_migration_transaction(
|transaction| {
let only_fetch_mvbox = transaction
.query_row(
"SELECT value FROM config WHERE keyname='only_fetch_mvbox'",
(),
|row| {
let value: String = row.get(0)?;
Ok(value)
},
)
.optional()?
.as_deref()
== Some("1");
if only_fetch_mvbox {
let mvbox_folder = transaction
.query_row(
"SELECT value FROM config WHERE keyname='configured_mvbox_folder'",
(),
|row| {
let value: String = row.get(0)?;
Ok(value)
},
)
.optional()?
.unwrap_or_else(|| "DeltaChat".to_string());
transaction.execute(
"UPDATE transports
SET entered_param=json_set(entered_param, '$.imap.folder', ?1),
configured_param=json_set(configured_param', '$.imap_folder', ?1)",
(mvbox_folder,),
)?;
}
Ok(())
},
sql.execute_migration(
"CREATE TABLE tls_spki (
host TEXT NOT NULL UNIQUE,
spki_hash TEXT NOT NULL -- base64 of SPKI SHA-256 hash
) STRICT",
migration_version,
)
.await?;

View File

@@ -413,6 +413,11 @@ https://delta.chat/donate"))]
#[strum(props(fallback = "Messages in this chat use classic email and are not encrypted."))]
ChatUnencryptedExplanation = 230,
#[strum(props(
fallback = "You are using the legacy option \"Settings → Advanced → Move automatically to DeltaChat Folder\".\n\nThis option will be removed in a few weeks and you should disable it already today.\n\nIf having chat messages mixed into your inbox is a problem, see https://delta.chat/legacy-move"
))]
MvboxMoveDeprecation = 231,
#[strum(props(fallback = "Outgoing audio call"))]
OutgoingAudioCall = 232,
@@ -1220,6 +1225,11 @@ pub(crate) fn chat_unencrypted_explanation(context: &Context) -> String {
translated(context, StockMessage::ChatUnencryptedExplanation)
}
/// Stock string: `You are using the legacy option "Move automatically to DeltaChat Folder`…
pub(crate) fn mvbox_move_deprecation(context: &Context) -> String {
translated(context, StockMessage::MvboxMoveDeprecation)
}
impl Viewtype {
/// returns Localized name for message viewtype
pub fn to_locale_string(&self, context: &Context) -> String {

View File

@@ -567,6 +567,7 @@ impl TestContext {
.unwrap();
ctx.set_config(Config::BccSelf, Some("1")).await.unwrap();
ctx.set_config(Config::SyncMsgs, Some("0")).await.unwrap();
ctx.set_config(Config::MvboxMove, Some("0")).await.unwrap();
Self {
ctx,

View File

@@ -19,7 +19,6 @@ use crate::config::Config;
use crate::configure::server_params::{ServerParams, expand_param_vector};
use crate::constants::{DC_LP_AUTH_FLAGS, DC_LP_AUTH_OAUTH2};
use crate::context::Context;
use crate::ensure_and_debug_assert;
use crate::events::EventType;
use crate::login_param::EnteredLoginParam;
use crate::net::load_connection_timestamp;
@@ -164,30 +163,22 @@ pub(crate) struct ConfiguredLoginParam {
/// `From:` address that was used at the time of configuration.
pub addr: String,
/// List of IMAP candidates to try.
pub imap: Vec<ConfiguredServerLoginParam>,
/// Custom IMAP user.
///
/// This overwrites autoconfig from the provider database
/// if non-empty.
// Custom IMAP user.
//
// This overwrites autoconfig from the provider database
// if non-empty.
pub imap_user: String,
pub imap_password: String,
// IMAP folder to watch.
//
// If not stored, should be interpreted as "INBOX".
// If stored, should be a folder name and not empty.
pub imap_folder: Option<String>,
/// List of SMTP candidates to try.
pub smtp: Vec<ConfiguredServerLoginParam>,
/// Custom SMTP user.
///
/// This overwrites autoconfig from the provider database
/// if non-empty.
// Custom SMTP user.
//
// This overwrites autoconfig from the provider database
// if non-empty.
pub smtp_user: String,
pub smtp_password: String,
@@ -208,13 +199,6 @@ pub(crate) struct ConfiguredLoginParam {
pub(crate) struct ConfiguredLoginParamJson {
pub addr: String,
pub imap: Vec<ConfiguredServerLoginParam>,
/// IMAP folder to watch.
///
/// Defaults to "INBOX" if unset.
#[serde(skip_serializing_if = "Option::is_none")]
pub imap_folder: Option<String>,
pub imap_user: String,
pub imap_password: String,
pub smtp: Vec<ConfiguredServerLoginParam>,
@@ -561,7 +545,6 @@ impl ConfiguredLoginParam {
Ok(Some(ConfiguredLoginParam {
addr,
imap,
imap_folder: None,
imap_user: mail_user,
imap_password: mail_pw,
smtp,
@@ -594,18 +577,11 @@ impl ConfiguredLoginParam {
pub(crate) fn from_json(json: &str) -> Result<Self> {
let json: ConfiguredLoginParamJson = serde_json::from_str(json)?;
ensure_and_debug_assert!(
json.imap_folder
.as_ref()
.is_none_or(|folder| !folder.is_empty()),
"Configured watched folder name cannot be empty"
);
let provider = json.provider_id.and_then(|id| get_provider_by_id(&id));
Ok(ConfiguredLoginParam {
addr: json.addr,
imap: json.imap,
imap_folder: json.imap_folder,
imap_user: json.imap_user,
imap_password: json.imap_password,
smtp: json.smtp,
@@ -643,7 +619,6 @@ impl From<ConfiguredLoginParam> for ConfiguredLoginParamJson {
imap: configured_login_param.imap,
imap_user: configured_login_param.imap_user,
imap_password: configured_login_param.imap_password,
imap_folder: configured_login_param.imap_folder,
smtp: configured_login_param.smtp,
smtp_user: configured_login_param.smtp_user,
smtp_password: configured_login_param.smtp_password,
@@ -663,16 +638,9 @@ pub(crate) async fn save_transport(
add_timestamp: i64,
is_published: bool,
) -> Result<bool> {
ensure_and_debug_assert!(
configured
.imap_folder
.as_ref()
.is_none_or(|folder| !folder.is_empty()),
"Configured watched folder name cannot be empty"
);
let addr = addr_normalize(&configured.addr);
let configured_addr = context.get_config(Config::ConfiguredAddr).await?;
let mut modified = context
.sql
.execute(

View File

@@ -34,7 +34,6 @@ async fn test_save_load_login_param() -> Result<()> {
},
user: "alice".to_string(),
}],
imap_folder: None,
imap_user: "".to_string(),
imap_password: "foo".to_string(),
smtp: vec![ConfiguredServerLoginParam {
@@ -143,7 +142,6 @@ async fn test_posteo_alias() -> Result<()> {
user: user.to_string(),
},
],
imap_folder: None,
imap_user: "alice@posteo.de".to_string(),
imap_password: "foobarbaz".to_string(),
smtp: vec![
@@ -274,7 +272,6 @@ fn dummy_configured_login_param(
},
user: addr.to_string(),
}],
imap_folder: None,
imap_user: addr.to_string(),
imap_password: "foobarbaz".to_string(),
smtp: vec![ConfiguredServerLoginParam {