Compare commits

..
Author SHA1 Message Date
Hocuri b5acd0844a Add some basic tests 2026-10-08 16:22:47 +02:00
Hocuri af755bff58 Show device message about removal of unencrypted messages
In its stead, remove `recode_avatar` high-level migration. It was added
5 years ago, and even if someone updates Delta Chat after such a long
time, large avatars are not a problem anymore since we don't send them
in unencrypted messages.
2026-10-08 16:19:29 +02:00
Hocuri dce1690e10 Migrate unencrypted chats to be read-only 2026-10-08 16:19:24 +02:00
9 changed files with 317 additions and 72 deletions
Generated
+19 -25
View File
@@ -272,19 +272,19 @@ dependencies = [
[[package]]
name = "async-imap"
version = "0.12.0"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f97b87216c9f0ccc63c516263169085fa34bba25633f6d96c6a9b9ca1fd5f70a"
checksum = "9a6728e0f7931b36d725ac234fcb02539e9f7888dbeaaa8a18d9ea5792181570"
dependencies = [
"async-channel 2.5.0",
"async-compression",
"base64 0.23.1",
"base64",
"bytes",
"chrono",
"futures-util",
"futures",
"imap-proto",
"log",
"nom 8.0.0",
"nom 7.1.3",
"pin-project",
"pin-utils",
"self_cell",
@@ -323,7 +323,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55219982f938e74491ba85dc4e49cefe8096b1e8f49348c67180a7d244988dca"
dependencies = [
"anyhow",
"base64 0.22.1",
"base64",
"futures",
"log",
"nom 8.0.0",
@@ -436,12 +436,6 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
[[package]]
name = "base64ct"
version = "1.6.0"
@@ -839,7 +833,7 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1f927b07c74ba84c7e5fe4db2baeb3e996ab2688992e39ac68ce3220a677c7e"
dependencies = [
"base64 0.22.1",
"base64",
"encoding_rs",
]
@@ -1344,7 +1338,7 @@ dependencies = [
"async-native-tls",
"async-smtp",
"async_zip",
"base64 0.23.1",
"base64",
"blake3",
"brotli",
"bytes",
@@ -1444,7 +1438,7 @@ version = "2.63.0-dev"
dependencies = [
"anyhow",
"async-channel 2.5.0",
"base64 0.23.1",
"base64",
"deltachat",
"deltachat-contact-tools",
"futures",
@@ -2946,11 +2940,11 @@ dependencies = [
[[package]]
name = "imap-proto"
version = "0.17.0"
version = "0.16.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ccf963d57074747b455398a1763d174da80bcaba6f51e3671a82252b531a68b"
checksum = "de555d9526462b6f9ece826a26fb7c67eca9a0245bd9ff84fa91972a5d5d8856"
dependencies = [
"nom 8.0.0",
"nom 7.1.3",
]
[[package]]
@@ -4186,7 +4180,7 @@ version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae"
dependencies = [
"base64 0.22.1",
"base64",
"serde",
]
@@ -4261,7 +4255,7 @@ dependencies = [
"aes-gcm",
"aes-kw",
"argon2",
"base64 0.22.1",
"base64",
"bitfields",
"block-padding",
"blowfish",
@@ -4540,7 +4534,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d6db66007eac4a0ec8331d0d20c734bd64f6445d64bbaf0d0a27fea7a054e36"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"derive_more 1.0.0",
"futures-lite",
@@ -5096,7 +5090,7 @@ version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"futures-core",
"futures-util",
@@ -5690,7 +5684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78db9c9912c90ea7487f49bc149b329b535806bfa12b740fbade73f573a3d9f"
dependencies = [
"aes",
"base64 0.22.1",
"base64",
"blake3",
"byte_string",
"bytes",
@@ -5990,7 +5984,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb921f10397d5669e1af6455e9e2d367bf1f9cebcd6b1dd1dc50e19f6a9ac2ac"
dependencies = [
"base64 0.22.1",
"base64",
"bounded-integer",
"byteorder",
"crc",
@@ -6405,7 +6399,7 @@ version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fcaf159b4e7a376b05b5bfd77bfd38f3324f5fce751b4213bfc7eaa47affb4e"
dependencies = [
"base64 0.22.1",
"base64",
"bytes",
"futures-core",
"futures-sink",
+2 -2
View File
@@ -43,7 +43,7 @@ ratelimit = { path = "./deltachat-ratelimit" }
anyhow = { workspace = true }
async-broadcast = "0.7.2"
async-channel = { workspace = true }
async-imap = { version = "0.12.0", default-features = false, features = ["runtime-tokio", "compress"] }
async-imap = { version = "0.11.3", default-features = false, features = ["runtime-tokio", "compress"] }
async-native-tls = { version = "0.6", default-features = false, features = ["runtime-tokio"] }
async-smtp = { version = "0.10.2", default-features = false, features = ["runtime-tokio"] }
async_zip = { version = "0.0.18", default-features = false, features = ["deflate", "tokio-fs"] }
@@ -178,7 +178,7 @@ harness = false
[workspace.dependencies]
anyhow = "1"
async-channel = "2.5.0"
base64 = { version = "0.23", default-features = false, features = ["std"] }
base64 = "0.22"
chrono = { version = "0.4.44", default-features = false }
deltachat-contact-tools = { path = "deltachat-contact-tools" }
deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
-1
View File
@@ -63,7 +63,6 @@ ignore = [
# Please keep this list alphabetically sorted.
skip = [
{ name = "async-channel", version = "1.9.0" },
{ name = "base64", version = "0.22.1" },
{ name = "bitflags", version = "1.3.2" },
{ name = "constant_time_eq", version = "0.3.1" },
{ name = "cpufeatures", version = "0.2.17" },
+21 -11
View File
@@ -1457,6 +1457,7 @@ impl Session {
fn drain_unsolicited_responses(&self, context: &Context) -> bool {
use UnsolicitedResponse::*;
use async_imap::imap_proto::Response;
use async_imap::imap_proto::ResponseCode;
let folder = self.selected_folder.as_deref().unwrap_or_default();
let mut should_refetch = false;
@@ -1471,19 +1472,28 @@ impl Session {
}
Expunge(_) | Recent(_) => {}
Other(ref response_data) => match response_data.parsed() {
Response::Fetch { .. } => {
info!(
context,
"Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
);
should_refetch = true;
}
Other(ref response_data) => {
match response_data.parsed() {
Response::Fetch { .. } => {
info!(
context,
"Need to refetch {folder:?}, got unsolicited FETCH {response:?}"
);
should_refetch = true;
}
_ => {
info!(context, "{folder:?}: got unsolicited response {response:?}")
// We are not interested in the following responses and they are are
// sent quite frequently, so, we ignore them without logging them.
Response::Done {
code: Some(ResponseCode::CopyUid(_, _, _)),
..
} => {}
_ => {
info!(context, "{folder:?}: got unsolicited response {response:?}")
}
}
},
}
_ => {
info!(context, "{folder:?}: got unsolicited response {response:?}")
}
+49 -22
View File
@@ -8,8 +8,7 @@ use anyhow::{Context as _, Result, bail};
use rusqlite::{Connection, OpenFlags, Row, config::DbConfig, types::ValueRef};
use tokio::sync::RwLock;
use crate::blob::BlobObject;
use crate::chat::ChatId;
use crate::chat::{ChatId, add_device_msg};
use crate::config::Config;
use crate::context::Context;
use crate::debug_logging::set_debug_logging_xdc;
@@ -214,32 +213,23 @@ impl Sql {
// this should be done before updates that use high-level objects that
// rely themselves on the low-level structure.
let recode_avatar = migrations::run(context, self)
let show_unencrypted_device_msg = migrations::run(context, self)
.await
.context("failed to run migrations")?;
// (2) updates that require high-level objects
// the structure is complete now and all objects are usable
if recode_avatar && let Some(avatar) = context.get_config(Config::Selfavatar).await? {
let mut blob = BlobObject::from_path(context, Path::new(&avatar))?;
match blob.recode_to_avatar_size(context).await {
Ok(()) => {
if let Some(path) = blob.to_abs_path().to_str() {
context
.set_config_internal(Config::Selfavatar, Some(path))
.await?;
} else {
warn!(context, "Setting selfavatar failed: non-UTF-8 filename");
}
}
Err(e) => {
warn!(context, "Migrations can't recode avatar, removing. {:#}", e);
context
.set_config_internal(Config::Selfavatar, None)
.await?
}
}
if show_unencrypted_device_msg {
let txt = r#"To keep Delta Chat simpler and more secure, it no longer sends or receives messages that aren't end-to-end encrypted.
Nothing is lost: your encrypted chats work as before, and old unencrypted chats remain readable.
To keep sending and receiving unencrypted email, use a regular email app. You can find your email password in Delta Chat under "Settings → Advanced → Relays": tap (or right-click) your address and choose "Edit Relay".
More details: https://..."#;
let mut msg = crate::message::Message::new_text(txt.to_string());
add_device_msg(context, Some("unencrypted-device-msg"), Some(&mut msg)).await?;
}
Ok(())
@@ -684,6 +674,43 @@ impl Sql {
}
}
pub(crate) trait TransactionExt {
/// Used for executing `SELECT COUNT` statements only. Returns the resulting count.
fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<usize>;
/// Executes a query which is expected to return one row and one
/// column. If the query does not return any rows, returns `Ok(None)`.
fn query_get_value<T>(
&self,
query: &str,
params: impl rusqlite::Params + Send,
) -> Result<Option<T>>
where
T: rusqlite::types::FromSql + Send + 'static;
}
impl TransactionExt for rusqlite::Transaction<'_> {
fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<usize> {
let count: isize = self.query_row(query, params, |row| row.get(0))?;
Ok(usize::try_from(count)?)
}
fn query_get_value<T>(
&self,
query: &str,
params: impl rusqlite::Params + Send,
) -> Result<Option<T>>
where
T: rusqlite::types::FromSql + Send + 'static,
{
match self.query_row(query, params, |row| row.get::<_, T>(0)) {
Ok(res) => Ok(Some(res)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(err) => Err(err.into()),
}
}
}
/// Creates a new SQLite connection.
///
/// `path` is the database path.
+122 -3
View File
@@ -16,6 +16,7 @@ use crate::key::DcKey;
use crate::log::warn;
use crate::sql::Sql;
use crate::sql::TransactionExt as _;
use crate::tools::{self, Time, inc_and_check, time_elapsed};
use crate::transport::ConfiguredLoginParam;
@@ -713,6 +714,111 @@ pub(crate) async fn msgs_to_key_contacts(context: &Context) -> Result<()> {
Ok(())
}
fn unencrypted_chats_migration(
context: &Context,
transaction: &mut rusqlite::Transaction<'_>,
) -> Result<bool> {
let mut show_unencrypted_device_msg = false;
// Migrate:
// - all unencrypted (i.e. ad-hoc) groups to have 0 members
// - all chats of type Mailinglist into a group with 0 members
transaction.execute_batch(
"
CREATE TEMP TABLE temp.legacy_unencrypted_chats(chat_id INTEGER PRIMARY KEY, type INTEGER) STRICT;
INSERT INTO temp.legacy_unencrypted_chats(chat_id, type)
SELECT id, type FROM chats
WHERE ((type=120 AND grpid='') OR type=140) -- Ad-hoc groups and mailinglists
AND id>9;
DELETE FROM chats_contacts
WHERE chat_id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats);
UPDATE chats SET type=120
WHERE id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats);
",
)?;
// Rewrite all address-contacts to have "Hidden" origin.
// We still need the contacts because we want to keep the messages, and every message needs a sender.
// Make sure that the address is available in the name, so that the user can still see it.
transaction.execute_batch(
"
UPDATE contacts
SET origin=8 -- Origin::Hidden
WHERE fingerprint='' AND id>9;
UPDATE contacts
SET name=name || ' (' || addr || ')'
WHERE fingerprint='' AND id>9 AND name!='';
UPDATE contacts
SET authname=authname || ' (' || addr || ')'
WHERE fingerprint='' AND id>9 AND name='' AND authname!='';
UPDATE contacts
SET authname=addr
WHERE fingerprint='' AND id>9 AND name='' AND authname='';
-- Also update the names of the chats:
UPDATE chats
SET name = (
SELECT CASE
WHEN c.name != '' THEN c.name
WHEN c.authname != '' THEN c.authname
ELSE c.addr
END
FROM chats_contacts cc
JOIN contacts c ON c.id = cc.contact_id
WHERE cc.chat_id = chats.id
)
WHERE type = 100 AND id > 9
AND EXISTS (
SELECT 1 FROM chats_contacts cc
JOIN contacts c ON c.id = cc.contact_id
WHERE cc.chat_id = chats.id AND c.fingerprint = '' AND c.id > 9
);
",
)?;
let legacy_chats =
transaction.count("SELECT COUNT(*) FROM temp.legacy_unencrypted_chats", ())?;
let legacy_contacts = transaction.count(
"SELECT COUNT(*) FROM contacts WHERE fingerprint='' AND id>9",
(),
)?;
if legacy_chats > 0 || legacy_contacts > 0 {
// Set the gray letter avatar for all legacy chats:
let blob = crate::blob::BlobObject::create_and_deduplicate_from_bytes(
context,
include_bytes!("../../assets/icon-unencrypted.png"),
"icon-unencrypted.png",
)?;
let new_param = &format!("i={}", blob.as_name());
transaction.execute(
"UPDATE chats SET param=? WHERE id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats)",
(new_param,),
)?;
transaction.execute(
"UPDATE contacts SET param=? WHERE fingerprint='' AND id>9",
(new_param,),
)?;
let force_encryption: Option<String> = transaction.query_get_value(
"SELECT value FROM config WHERE keyname='force_encryption'",
(),
)?;
if force_encryption == Some("0".to_string()) {
show_unencrypted_device_msg = true;
}
}
transaction.execute("DROP TABLE temp.legacy_unencrypted_chats", ())?;
Ok(show_unencrypted_device_msg)
}
impl Sql {
async fn set_db_version(&self, version: i32) -> Result<()> {
self.set_raw_config_int(VERSION_CFG, version).await?;
@@ -812,7 +918,7 @@ pub async fn run(context: &Context, sql: &Sql) -> Result<bool> {
}
let dbversion = dbversion_before_update;
let mut recode_avatar = false;
let mut show_unencrypted_device_msg = false;
if dbversion < 1 {
sql.execute_migration(
@@ -1187,7 +1293,7 @@ CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0,
.await?;
}
if dbversion < 77 {
recode_avatar = true;
// removed
sql.set_db_version(77).await?;
}
if dbversion < 78 {
@@ -2687,6 +2793,19 @@ CREATE TABLE smtp_success (
.await?;
}
inc_and_check(&mut migration_version, 169)?;
if dbversion < migration_version {
sql.execute_migration_transaction(
|transaction| {
show_unencrypted_device_msg = unencrypted_chats_migration(context, transaction)?;
Ok(())
},
migration_version,
)
.await?;
}
let new_version = sql
.get_raw_config_int(VERSION_CFG)
.await?
@@ -2701,7 +2820,7 @@ CREATE TABLE smtp_success (
}
info!(context, "Database version: v{new_version}.");
Ok(recode_avatar)
Ok(show_unencrypted_device_msg)
}
#[cfg(test)]
+101 -2
View File
@@ -1,8 +1,10 @@
use super::*;
use crate::chat;
use crate::chat::Chat;
use crate::chat::ChatId;
use crate::config::Config;
use crate::constants;
use crate::constants::Chattype;
use crate::contact::Contact;
use crate::contact::ContactId;
use crate::contact::Origin;
@@ -145,7 +147,8 @@ async fn test_key_contacts_migration_email1() -> Result<()> {
.unwrap();
let email_bob = Contact::get_by_id(&t, email_bob_id).await?;
assert_eq!(email_bob.is_key_contact(), false);
assert_eq!(email_bob.origin, Origin::OutgoingTo);
// All email address contacts are hidden now:
assert_eq!(email_bob.origin, Origin::Hidden);
assert_eq!(email_bob.e2ee_avail(&t).await?, false);
assert_eq!(email_bob.fingerprint(), None);
@@ -178,7 +181,8 @@ async fn test_key_contacts_migration_email2() -> Result<()> {
.unwrap();
let email_bob = Contact::get_by_id(&t, email_bob_id).await?;
assert_eq!(email_bob.is_key_contact(), false);
assert_eq!(email_bob.origin, Origin::OutgoingTo);
// All email address contacts are hidden now:
assert_eq!(email_bob.origin, Origin::Hidden);
assert_eq!(email_bob.e2ee_avail(&t).await?, false);
assert_eq!(email_bob.fingerprint(), None);
@@ -227,3 +231,98 @@ async fn test_key_contacts_migration_verified() -> Result<()> {
Ok(())
}
/// Creates a context right before the unencrypted-chats migration (v169).
async fn context_before_unencrypted_migration() -> TestContext {
STOP_MIGRATIONS_AT
.scope(168, async move { TestContext::new_alice().await })
.await
}
/// Adds legacy data: a 1:1 chat (chat 10), an ad-hoc group (chat 11),
/// a mailing list (chat 12) and an address-contact (contact 10).
async fn add_legacy_data(t: &TestContext) -> Result<()> {
t.sql
.call_write(|conn| {
conn.execute_batch(
r#"
INSERT INTO contacts (id, name, addr, origin, fingerprint)
VALUES (10, 'Bob', 'bob@example.net', 16384, '');
INSERT INTO chats (id, type, name, grpid) VALUES
(10, 100, 'Bob', ''),
(11, 120, 'Thread', ''),
(12, 140, 'List', 'list.example.org');
INSERT INTO chats_contacts (chat_id, contact_id) VALUES
(10, 10), (11, 1), (11, 10), (12, 10);"#,
)?;
Ok(())
})
.await
}
/// Legacy unencrypted 1:1 chats get the email address put into the name,
/// and get the "unencrypted" avatar.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unencrypted_chats_migration_1to1_chat() -> Result<()> {
let t = &context_before_unencrypted_migration().await;
add_legacy_data(t).await?;
t.sql.run_migrations(t).await?;
let bob = Contact::get_by_id(t, ContactId::new(10)).await?;
assert_eq!(bob.get_display_name(), "Bob (bob@example.net)");
assert!(bob.get_profile_image(t).await?.unwrap().exists());
let chat = Chat::load_from_db(t, ChatId::new(10)).await?;
assert_eq!(chat.get_name(), "Bob (bob@example.net)");
assert!(chat.get_profile_image(t).await?.unwrap().exists());
Ok(())
}
/// Legacy ad-hoc groups and mailing lists become read-only groups without members
/// and get the "unencrypted" avatar.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unencrypted_chats_migration_readonly_groups() -> Result<()> {
let t = &context_before_unencrypted_migration().await;
add_legacy_data(t).await?;
t.sql.run_migrations(t).await?;
for chat_id in [11, 12] {
let chat_id = ChatId::new(chat_id);
let chat = Chat::load_from_db(t, chat_id).await?;
assert_eq!(chat.typ, Chattype::Group);
assert!(chat::get_chat_contacts(t, chat_id).await?.is_empty());
assert_eq!(chat.can_send(t).await?, false);
assert!(chat.get_profile_image(t).await?.unwrap().exists());
}
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_unencrypted_chats_migration_device_msg() -> Result<()> {
for (force_encryption, has_legacy_data, expect_device_msg) in [
(Some("0"), true, true),
(Some("1"), true, false),
(None, true, false),
(Some("0"), false, false),
] {
let t = &context_before_unencrypted_migration().await;
t.sql
.set_raw_config("force_encryption", force_encryption)
.await?;
if has_legacy_data {
add_legacy_data(t).await?;
}
t.sql.run_migrations(t).await?;
let shown = chat::was_device_msg_ever_added(t, "unencrypted-device-msg").await?;
assert_eq!(shown, expect_device_msg);
// This assert will need to be removed once we remove the ForceEncryption config,
// but it is useful for now to check that the logic is implemented correctly:
let allow_unencrypted = !t.get_config_bool(Config::ForceEncryption).await?;
assert_eq!(shown, allow_unencrypted && has_legacy_data);
}
Ok(())
}
+1 -4
View File
@@ -105,10 +105,7 @@ async fn check_aeap_transition(chat_for_transition: ChatForTransition) {
check_that_transition_worked(bob, &groups, alice_contact, ALICE_NEW_ADDR).await;
tcm.section("Test switching back");
alice
.set_primary_self_addr("alice@example.org")
.await
.unwrap();
tcm.change_addr(alice, "alice@example.org").await;
let sent = alice
.send_text(chat_to_send, "Hello from my old addr!")
.await;
+2 -2
View File
@@ -740,14 +740,14 @@ pub(crate) fn maybe_update_sending_transport(
pub async fn add_pseudo_transport(context: &Context, addr: &str) -> Result<()> {
context.sql
.execute(
"INSERT INTO transports (addr, entered_param, configured_param) VALUES (?, ?, ?)",
"INSERT OR IGNORE INTO transports (addr, entered_param, configured_param) VALUES (?, ?, ?)",
(
addr,
serde_json::to_string(&EnteredLoginParam{addr: addr.to_string(), ..Default::default()})?,
format!(r#"{{"addr":"{addr}","imap":[],"imap_user":"","imap_password":"","smtp":[],"smtp_user":"","smtp_password":"","certificate_checks":"Automatic","oauth2":false}}"#)
),
)
.await.with_context(|| format!("Failed to insert pseudo transport for {addr:?}"))?;
.await?;
Ok(())
}