feat: Remove hidden relays automatically

Automatically remove relays that are hidden (`is_published=0`)
and haven't been used to receive new messages for over 90 days.

Closes: #8384
Signed-off-by: Jagoda Ślązak <jslazak@jslazak.com>

src/sql.rs

@@ -47,6 +47,9 @@ mod pool;
 
 use pool::{Pool, WalCheckpointStats};
 
+/// How long a hidden and unused transport should be kept in the database before being deleted.
+pub const HIDDEN_TRANSPORT_CLEANUP_TIME: i64 = 90 * 24 * 60 * 60;
+
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(Debug)]
 pub struct Sql {
@@ -908,10 +911,34 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         .log_err(context)
         .ok();
 
+    remove_unused_hidden_transports(context)
+        .await
+        .context("failed to remove unused hidden transports")
+        .log_err(context)
+        .ok();
+
     info!(context, "Housekeeping done.");
     Ok(())
 }
 
+/// Removes transports that are hidden (`is_published=0`),
+/// and haven't been used to receive new messages for [`HIDDEN_TRANSPORT_CLEANUP_TIME`] seconds.
+pub async fn remove_unused_hidden_transports(context: &Context) -> Result<usize> {
+    let now = time();
+    let cutoff = now.saturating_sub(HIDDEN_TRANSPORT_CLEANUP_TIME);
+    context
+        .sql
+        .execute(
+            "DELETE FROM transports \
+            WHERE is_published=0 AND id NOT IN \
+            (SELECT imap.transport_id FROM imap \
+            INNER JOIN msgs ON msgs.rfc724_mid=imap.rfc724_mid \
+            WHERE msgs.timestamp_rcvd>?)",
+            (cutoff,),
+        )
+        .await
+}
+
 /// Get the value of a column `idx` of the `row` as `Vec<u8>`.
 pub fn row_get_vec(row: &Row, idx: usize) -> rusqlite::Result<Vec<u8>> {
     row.get(idx).or_else(|err| match row.get_ref(idx)? {

src/sql/sql_tests.rs

@@ -367,3 +367,115 @@ async fn test_incremental_vacuum() -> Result<()> {
 
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_remove_unused_hidden_transports() -> Result<()> {
+    let t = TestContext::new().await;
+    let now = time();
+
+    let transport_id = t
+        .sql
+        .transaction(|t| {
+            // published transport
+            _ = t.query_row(
+                "INSERT INTO transports \
+                (addr, \
+                entered_param, \
+                configured_param) \
+                VALUES (?, ?, ?) \
+                RETURNING 1",
+                ("1", "", ""),
+                |_| Ok(()),
+            )?;
+
+            // not published transport
+            let transport_id = t.query_row::<u64, _, _>(
+                "INSERT INTO transports \
+                (addr, \
+                entered_param, \
+                configured_param, \
+                is_published) \
+                VALUES (?, ?, ?, ?) \
+                RETURNING id",
+                ("2", "", "", 0),
+                |row| Ok(row.get(0)?),
+            )?;
+
+            // msgs
+            _ = t.query_row(
+                "INSERT INTO msgs \
+                (rfc724_mid, \
+                timestamp_rcvd, \
+                mime_headers, \
+                mime_in_reply_to, \
+                mime_references, \
+                txt_normalized) \
+                VALUES (?, ?, ?, ?, ?, ?) \
+                RETURNING 1",
+                ("smth", now, "", "", "", ""),
+                |_| Ok(()),
+            )?;
+
+            // imap
+            _ = t.query_row(
+                "INSERT INTO imap \
+                (transport_id, \
+                rfc724_mid, \
+                folder, \
+                target, \
+                uid, \
+                uidvalidity) \
+                VALUES (?, ?, ?, ?, ?, ?) \
+                RETURNING 1",
+                (transport_id, "smth", "", "", 0, 0),
+                |_| Ok(()),
+            )?;
+
+            Ok(transport_id)
+        })
+        .await?;
+
+    macro_rules! assert_transport_removed {
+        ($removed:expr) => {
+            let transports_count: u32 = t
+                .sql
+                .query_get_value("SELECT count(*) FROM transports", ())
+                .await?
+                .unwrap();
+            assert_eq!(transports_count, if $removed { 1 } else { 2 });
+            let hidden_transport_present: bool = t
+                .sql
+                .query_get_value(
+                    "SELECT count(*) FROM transports WHERE id=?",
+                    (transport_id,),
+                )
+                .await?
+                .unwrap();
+            assert_eq!(hidden_transport_present, !$removed);
+        };
+    }
+
+    assert_transport_removed!(false);
+    remove_unused_hidden_transports(&t).await?;
+    // it was recently used, so nothing is deleted
+    assert_transport_removed!(false);
+
+    // move the message back in time
+    let msg_time = now - HIDDEN_TRANSPORT_CLEANUP_TIME - 1;
+    t.sql
+        .transaction(|t| {
+            let affected = t.query_row::<u64, _, _>(
+                "UPDATE msgs SET timestamp_rcvd=? WHERE rfc724_mid='smth' RETURNING 1",
+                (msg_time,),
+                |row| Ok(row.get(0)?),
+            )?;
+            assert_eq!(affected, 1);
+            Ok(())
+        })
+        .await?;
+
+    remove_unused_hidden_transports(&t).await?;
+    assert_transport_removed!(true);
+
+    Ok(())
+}