Add some basic tests

In its stead, remove `recode_avatar` high-level migration. It was added
5 years ago, and even if someone updates Delta Chat after such a long
time, large avatars are not a problem anymore since we don't send them
in unencrypted messages.

src/sql.rs

@@ -8,8 +8,7 @@ use anyhow::{Context as _, Result, bail};
 use rusqlite::{Connection, OpenFlags, Row, config::DbConfig, types::ValueRef};
 use tokio::sync::RwLock;
 
-use crate::blob::BlobObject;
-use crate::chat::ChatId;
+use crate::chat::{ChatId, add_device_msg};
 use crate::config::Config;
 use crate::context::Context;
 use crate::debug_logging::set_debug_logging_xdc;
@@ -214,32 +213,23 @@ impl Sql {
         // this should be done before updates that use high-level objects that
         // rely themselves on the low-level structure.
 
-        let recode_avatar = migrations::run(context, self)
+        let show_unencrypted_device_msg = migrations::run(context, self)
             .await
             .context("failed to run migrations")?;
 
         // (2) updates that require high-level objects
         // the structure is complete now and all objects are usable
 
-        if recode_avatar && let Some(avatar) = context.get_config(Config::Selfavatar).await? {
-            let mut blob = BlobObject::from_path(context, Path::new(&avatar))?;
-            match blob.recode_to_avatar_size(context).await {
-                Ok(()) => {
-                    if let Some(path) = blob.to_abs_path().to_str() {
-                        context
-                            .set_config_internal(Config::Selfavatar, Some(path))
-                            .await?;
-                    } else {
-                        warn!(context, "Setting selfavatar failed: non-UTF-8 filename");
-                    }
-                }
-                Err(e) => {
-                    warn!(context, "Migrations can't recode avatar, removing. {:#}", e);
-                    context
-                        .set_config_internal(Config::Selfavatar, None)
-                        .await?
-                }
-            }
+        if show_unencrypted_device_msg {
+            let txt = r#"To keep Delta Chat simpler and more secure, it no longer sends or receives messages that aren't end-to-end encrypted.
+
+Nothing is lost: your encrypted chats work as before, and old unencrypted chats remain readable.
+
+To keep sending and receiving unencrypted email, use a regular email app. You can find your email password in Delta Chat under "Settings → Advanced → Relays": tap (or right-click) your address and choose "Edit Relay".
+
+More details: https://..."#;
+            let mut msg = crate::message::Message::new_text(txt.to_string());
+            add_device_msg(context, Some("unencrypted-device-msg"), Some(&mut msg)).await?;
         }
 
         Ok(())
@@ -684,6 +674,43 @@ impl Sql {
     }
 }
 
+pub(crate) trait TransactionExt {
+    /// Used for executing `SELECT COUNT` statements only. Returns the resulting count.
+    fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<usize>;
+
+    /// Executes a query which is expected to return one row and one
+    /// column. If the query does not return any rows, returns `Ok(None)`.
+    fn query_get_value<T>(
+        &self,
+        query: &str,
+        params: impl rusqlite::Params + Send,
+    ) -> Result<Option<T>>
+    where
+        T: rusqlite::types::FromSql + Send + 'static;
+}
+
+impl TransactionExt for rusqlite::Transaction<'_> {
+    fn count(&self, query: &str, params: impl rusqlite::Params + Send) -> Result<usize> {
+        let count: isize = self.query_row(query, params, |row| row.get(0))?;
+        Ok(usize::try_from(count)?)
+    }
+
+    fn query_get_value<T>(
+        &self,
+        query: &str,
+        params: impl rusqlite::Params + Send,
+    ) -> Result<Option<T>>
+    where
+        T: rusqlite::types::FromSql + Send + 'static,
+    {
+        match self.query_row(query, params, |row| row.get::<_, T>(0)) {
+            Ok(res) => Ok(Some(res)),
+            Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
+            Err(err) => Err(err.into()),
+        }
+    }
+}
+
 /// Creates a new SQLite connection.
 ///
 /// `path` is the database path.

src/sql/migrations.rs

@@ -16,6 +16,7 @@ use crate::key::DcKey;
 use crate::log::warn;
 
 use crate::sql::Sql;
+use crate::sql::TransactionExt as _;
 use crate::tools::{self, Time, inc_and_check, time_elapsed};
 use crate::transport::ConfiguredLoginParam;
 
@@ -713,6 +714,111 @@ pub(crate) async fn msgs_to_key_contacts(context: &Context) -> Result<()> {
     Ok(())
 }
 
+fn unencrypted_chats_migration(
+    context: &Context,
+    transaction: &mut rusqlite::Transaction<'_>,
+) -> Result<bool> {
+    let mut show_unencrypted_device_msg = false;
+
+    // Migrate:
+    // - all unencrypted (i.e. ad-hoc) groups to have 0 members
+    // - all chats of type Mailinglist into a group with 0 members
+    transaction.execute_batch(
+        "
+CREATE TEMP TABLE temp.legacy_unencrypted_chats(chat_id INTEGER PRIMARY KEY, type INTEGER) STRICT;
+
+INSERT INTO temp.legacy_unencrypted_chats(chat_id, type)
+SELECT id, type FROM chats
+WHERE ((type=120 AND grpid='') OR type=140) -- Ad-hoc groups and mailinglists
+  AND id>9;
+
+DELETE FROM chats_contacts
+WHERE chat_id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats);
+
+UPDATE chats SET type=120
+WHERE id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats);
+",
+    )?;
+
+    // Rewrite all address-contacts to have "Hidden" origin.
+    // We still need the contacts because we want to keep the messages, and every message needs a sender.
+    // Make sure that the address is available in the name, so that the user can still see it.
+    transaction.execute_batch(
+        "
+UPDATE contacts
+SET origin=8 -- Origin::Hidden
+WHERE fingerprint='' AND id>9;
+
+UPDATE contacts
+SET name=name || ' (' || addr || ')'
+WHERE fingerprint='' AND id>9 AND name!='';
+
+UPDATE contacts
+SET authname=authname || ' (' || addr || ')'
+WHERE fingerprint='' AND id>9 AND name='' AND authname!='';
+
+UPDATE contacts
+SET authname=addr
+WHERE fingerprint='' AND id>9 AND name='' AND authname='';
+
+-- Also update the names of the chats:
+UPDATE chats
+SET name = (
+    SELECT CASE
+        WHEN c.name != ''     THEN c.name
+        WHEN c.authname != '' THEN c.authname
+        ELSE c.addr
+    END
+    FROM chats_contacts cc
+    JOIN contacts c ON c.id = cc.contact_id
+    WHERE cc.chat_id = chats.id
+)
+WHERE type = 100 AND id > 9
+AND EXISTS (
+    SELECT 1 FROM chats_contacts cc
+    JOIN contacts c ON c.id = cc.contact_id
+    WHERE cc.chat_id = chats.id AND c.fingerprint = '' AND c.id > 9
+);
+",
+    )?;
+
+    let legacy_chats =
+        transaction.count("SELECT COUNT(*) FROM temp.legacy_unencrypted_chats", ())?;
+    let legacy_contacts = transaction.count(
+        "SELECT COUNT(*) FROM contacts WHERE fingerprint='' AND id>9",
+        (),
+    )?;
+    if legacy_chats > 0 || legacy_contacts > 0 {
+        // Set the gray letter avatar for all legacy chats:
+        let blob = crate::blob::BlobObject::create_and_deduplicate_from_bytes(
+            context,
+            include_bytes!("../../assets/icon-unencrypted.png"),
+            "icon-unencrypted.png",
+        )?;
+        let new_param = &format!("i={}", blob.as_name());
+        transaction.execute(
+            "UPDATE chats SET param=? WHERE id IN (SELECT chat_id FROM temp.legacy_unencrypted_chats)",
+            (new_param,),
+        )?;
+        transaction.execute(
+            "UPDATE contacts SET param=? WHERE fingerprint='' AND id>9",
+            (new_param,),
+        )?;
+
+        let force_encryption: Option<String> = transaction.query_get_value(
+            "SELECT value FROM config WHERE keyname='force_encryption'",
+            (),
+        )?;
+        if force_encryption == Some("0".to_string()) {
+            show_unencrypted_device_msg = true;
+        }
+    }
+
+    transaction.execute("DROP TABLE temp.legacy_unencrypted_chats", ())?;
+
+    Ok(show_unencrypted_device_msg)
+}
+
 impl Sql {
     async fn set_db_version(&self, version: i32) -> Result<()> {
         self.set_raw_config_int(VERSION_CFG, version).await?;
@@ -812,7 +918,7 @@ pub async fn run(context: &Context, sql: &Sql) -> Result<bool> {
     }
 
     let dbversion = dbversion_before_update;
-    let mut recode_avatar = false;
+    let mut show_unencrypted_device_msg = false;
 
     if dbversion < 1 {
         sql.execute_migration(
@@ -1187,7 +1293,7 @@ CREATE TABLE imap_sync (folder TEXT PRIMARY KEY, uidvalidity INTEGER DEFAULT 0,
             .await?;
     }
     if dbversion < 77 {
-        recode_avatar = true;
+        // removed
         sql.set_db_version(77).await?;
     }
     if dbversion < 78 {
@@ -2687,6 +2793,19 @@ CREATE TABLE smtp_success (
         .await?;
     }
 
+    inc_and_check(&mut migration_version, 169)?;
+    if dbversion < migration_version {
+        sql.execute_migration_transaction(
+            |transaction| {
+                show_unencrypted_device_msg = unencrypted_chats_migration(context, transaction)?;
+
+                Ok(())
+            },
+            migration_version,
+        )
+        .await?;
+    }
+
     let new_version = sql
         .get_raw_config_int(VERSION_CFG)
         .await?
@@ -2701,7 +2820,7 @@ CREATE TABLE smtp_success (
     }
     info!(context, "Database version: v{new_version}.");
 
-    Ok(recode_avatar)
+    Ok(show_unencrypted_device_msg)
 }
 
 #[cfg(test)]

src/sql/migrations/migrations_tests.rs

@@ -1,8 +1,10 @@
 use super::*;
 use crate::chat;
+use crate::chat::Chat;
 use crate::chat::ChatId;
 use crate::config::Config;
 use crate::constants;
+use crate::constants::Chattype;
 use crate::contact::Contact;
 use crate::contact::ContactId;
 use crate::contact::Origin;
@@ -145,7 +147,8 @@ async fn test_key_contacts_migration_email1() -> Result<()> {
         .unwrap();
     let email_bob = Contact::get_by_id(&t, email_bob_id).await?;
     assert_eq!(email_bob.is_key_contact(), false);
-    assert_eq!(email_bob.origin, Origin::OutgoingTo);
+    // All email address contacts are hidden now:
+    assert_eq!(email_bob.origin, Origin::Hidden);
     assert_eq!(email_bob.e2ee_avail(&t).await?, false);
     assert_eq!(email_bob.fingerprint(), None);
 
@@ -178,7 +181,8 @@ async fn test_key_contacts_migration_email2() -> Result<()> {
         .unwrap();
     let email_bob = Contact::get_by_id(&t, email_bob_id).await?;
     assert_eq!(email_bob.is_key_contact(), false);
-    assert_eq!(email_bob.origin, Origin::OutgoingTo);
+    // All email address contacts are hidden now:
+    assert_eq!(email_bob.origin, Origin::Hidden);
     assert_eq!(email_bob.e2ee_avail(&t).await?, false);
     assert_eq!(email_bob.fingerprint(), None);
 
@@ -227,3 +231,98 @@ async fn test_key_contacts_migration_verified() -> Result<()> {
 
     Ok(())
 }
+
+/// Creates a context right before the unencrypted-chats migration (v169).
+async fn context_before_unencrypted_migration() -> TestContext {
+    STOP_MIGRATIONS_AT
+        .scope(168, async move { TestContext::new_alice().await })
+        .await
+}
+
+/// Adds legacy data: a 1:1 chat (chat 10), an ad-hoc group (chat 11),
+/// a mailing list (chat 12) and an address-contact (contact 10).
+async fn add_legacy_data(t: &TestContext) -> Result<()> {
+    t.sql
+        .call_write(|conn| {
+            conn.execute_batch(
+                r#"
+INSERT INTO contacts (id, name, addr, origin, fingerprint)
+    VALUES (10, 'Bob', 'bob@example.net', 16384, '');
+INSERT INTO chats (id, type, name, grpid) VALUES
+    (10, 100, 'Bob', ''),
+    (11, 120, 'Thread', ''),
+    (12, 140, 'List', 'list.example.org');
+INSERT INTO chats_contacts (chat_id, contact_id) VALUES
+    (10, 10), (11, 1), (11, 10), (12, 10);"#,
+            )?;
+            Ok(())
+        })
+        .await
+}
+
+/// Legacy unencrypted 1:1 chats get the email address put into the name,
+/// and get the "unencrypted" avatar.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_unencrypted_chats_migration_1to1_chat() -> Result<()> {
+    let t = &context_before_unencrypted_migration().await;
+    add_legacy_data(t).await?;
+    t.sql.run_migrations(t).await?;
+
+    let bob = Contact::get_by_id(t, ContactId::new(10)).await?;
+    assert_eq!(bob.get_display_name(), "Bob (bob@example.net)");
+    assert!(bob.get_profile_image(t).await?.unwrap().exists());
+
+    let chat = Chat::load_from_db(t, ChatId::new(10)).await?;
+    assert_eq!(chat.get_name(), "Bob (bob@example.net)");
+    assert!(chat.get_profile_image(t).await?.unwrap().exists());
+
+    Ok(())
+}
+
+/// Legacy ad-hoc groups and mailing lists become read-only groups without members
+/// and get the "unencrypted" avatar.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_unencrypted_chats_migration_readonly_groups() -> Result<()> {
+    let t = &context_before_unencrypted_migration().await;
+    add_legacy_data(t).await?;
+    t.sql.run_migrations(t).await?;
+
+    for chat_id in [11, 12] {
+        let chat_id = ChatId::new(chat_id);
+        let chat = Chat::load_from_db(t, chat_id).await?;
+        assert_eq!(chat.typ, Chattype::Group);
+        assert!(chat::get_chat_contacts(t, chat_id).await?.is_empty());
+        assert_eq!(chat.can_send(t).await?, false);
+        assert!(chat.get_profile_image(t).await?.unwrap().exists());
+    }
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_unencrypted_chats_migration_device_msg() -> Result<()> {
+    for (force_encryption, has_legacy_data, expect_device_msg) in [
+        (Some("0"), true, true),
+        (Some("1"), true, false),
+        (None, true, false),
+        (Some("0"), false, false),
+    ] {
+        let t = &context_before_unencrypted_migration().await;
+        t.sql
+            .set_raw_config("force_encryption", force_encryption)
+            .await?;
+        if has_legacy_data {
+            add_legacy_data(t).await?;
+        }
+        t.sql.run_migrations(t).await?;
+
+        let shown = chat::was_device_msg_ever_added(t, "unencrypted-device-msg").await?;
+        assert_eq!(shown, expect_device_msg);
+
+        // This assert will need to be removed once we remove the ForceEncryption config,
+        // but it is useful for now to check that the logic is implemented correctly:
+        let allow_unencrypted = !t.get_config_bool(Config::ForceEncryption).await?;
+        assert_eq!(shown, allow_unencrypted && has_legacy_data);
+    }
+    Ok(())
+}